extent-tree.c 232 KB
Newer Older
Chris Mason's avatar
Chris Mason committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Zach Brown's avatar
Zach Brown committed
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
Chris Mason's avatar
Chris Mason committed
27
#include "compat.h"
28
#include "hash.h"
29 30 31
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
32
#include "transaction.h"
33
#include "volumes.h"
David Woodhouse's avatar
David Woodhouse committed
34
#include "raid56.h"
35
#include "locking.h"
36
#include "free-space-cache.h"
37
#include "math.h"
38

39 40
#undef SCRAMBLE_DELAYED_REFS

41 42
/*
 * control flags for do_chunk_alloc's force field
43 44 45 46 47 48 49 50 51
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
52 53
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
54 55 56
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
57 58
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
59 60
};

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

76
static int update_block_group(struct btrfs_root *root,
77
			      u64 bytenr, u64 num_bytes, int alloc);
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
97
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
98 99
			  struct btrfs_root *extent_root, u64 flags,
			  int force);
100 101
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
Josef Bacik's avatar
Josef Bacik committed
102 103
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
104 105
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
106 107
static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes);
108 109
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved);
110

111 112 113 114 115 116 117
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

118 119 120 121 122
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

123
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
124 125 126 127 128 129
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
130 131 132
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
133
		kfree(cache->free_space_ctl);
134
		kfree(cache);
135
	}
136 137
}

138 139 140 141
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
142
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
169 170 171 172

	if (info->first_logical_byte > block_group->key.objectid)
		info->first_logical_byte = block_group->key.objectid;

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
214
	if (ret) {
215
		btrfs_get_block_group(ret);
216 217 218
		if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
			info->first_logical_byte = ret->key.objectid;
	}
219 220 221 222 223
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

224 225
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
226
{
227 228 229 230 231 232 233
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
234

235 236 237 238
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
239

240 241 242 243 244 245 246
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
247 248
}

249 250
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
251 252 253 254 255 256
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

257 258 259 260 261
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
262 263
		if (ret)
			return ret;
264 265
	}

266 267 268 269 270
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
271 272
		if (ret)
			return ret;
273

274
		while (nr--) {
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
			u64 start, len;

			if (logical[nr] > cache->key.objectid +
			    cache->key.offset)
				continue;

			if (logical[nr] + stripe_len <= cache->key.objectid)
				continue;

			start = logical[nr];
			if (start < cache->key.objectid) {
				start = cache->key.objectid;
				len = (logical[nr] + stripe_len) - start;
			} else {
				len = min_t(u64, stripe_len,
					    cache->key.objectid +
					    cache->key.offset - start);
			}

			cache->bytes_super += len;
			ret = add_excluded_extent(root, start, len);
296 297 298 299
			if (ret) {
				kfree(logical);
				return ret;
			}
300
		}
301

302 303 304 305 306
		kfree(logical);
	}
	return 0;
}

307 308 309 310 311 312 313 314 315 316 317
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

318 319 320
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

336 337 338 339 340
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
341
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
342 343
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
344
	u64 extent_start, extent_end, size, total_added = 0;
345 346 347
	int ret;

	while (start < end) {
348
		ret = find_first_extent_bit(info->pinned_extents, start,
349
					    &extent_start, &extent_end,
350 351
					    EXTENT_DIRTY | EXTENT_UPTODATE,
					    NULL);
352 353 354
		if (ret)
			break;

355
		if (extent_start <= start) {
356 357 358
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
359
			total_added += size;
360 361
			ret = btrfs_add_free_space(block_group, start,
						   size);
362
			BUG_ON(ret); /* -ENOMEM or logic error */
363 364 365 366 367 368 369 370
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
371
		total_added += size;
372
		ret = btrfs_add_free_space(block_group, start, size);
373
		BUG_ON(ret); /* -ENOMEM or logic error */
374 375
	}

376
	return total_added;
377 378
}

379
static noinline void caching_thread(struct btrfs_work *work)
380
{
381 382 383 384
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
385
	struct btrfs_path *path;
386
	struct extent_buffer *leaf;
387
	struct btrfs_key key;
388
	u64 total_found = 0;
389 390 391
	u64 last = 0;
	u32 nritems;
	int ret = 0;
392

393 394 395 396 397
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

398 399
	path = btrfs_alloc_path();
	if (!path)
400
		goto out;
401

402
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
403

404
	/*
405 406 407 408
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
409 410
	 */
	path->skip_locking = 1;
411
	path->search_commit_root = 1;
412
	path->reada = 1;
413

Yan Zheng's avatar
Yan Zheng committed
414
	key.objectid = last;
415
	key.offset = 0;
416
	key.type = BTRFS_EXTENT_ITEM_KEY;
417
again:
418
	mutex_lock(&caching_ctl->mutex);
419 420 421
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

422
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
423
	if (ret < 0)
424
		goto err;
Yan Zheng's avatar
Yan Zheng committed
425

426 427 428
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

429
	while (1) {
430
		if (btrfs_fs_closing(fs_info) > 1) {
431
			last = (u64)-1;
432
			break;
433
		}
434

435 436 437 438 439
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
440
				break;
441

442
			if (need_resched()) {
443
				caching_ctl->progress = last;
Chris Mason's avatar
Chris Mason committed
444
				btrfs_release_path(path);
445 446
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
447
				cond_resched();
448 449
				goto again;
			}
450 451 452 453 454 455

			ret = btrfs_next_leaf(extent_root, path);
			if (ret < 0)
				goto err;
			if (ret)
				break;
456 457 458
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
459
		}
460

461 462
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
463
			continue;
464
		}
465

466
		if (key.objectid >= block_group->key.objectid +
467
		    block_group->key.offset)
468
			break;
469

470 471
		if (key.type == BTRFS_EXTENT_ITEM_KEY ||
		    key.type == BTRFS_METADATA_ITEM_KEY) {
472 473 474
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
475 476 477 478 479
			if (key.type == BTRFS_METADATA_ITEM_KEY)
				last = key.objectid +
					fs_info->tree_root->leafsize;
			else
				last = key.objectid + key.offset;
480

481 482 483 484
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
485
		}
486 487
		path->slots[0]++;
	}
488
	ret = 0;
489

490 491 492
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
493
	caching_ctl->progress = (u64)-1;
494 495

	spin_lock(&block_group->lock);
496
	block_group->caching_ctl = NULL;
497 498
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
499

500
err:
501
	btrfs_free_path(path);
502
	up_read(&fs_info->extent_commit_sem);
503

504 505 506
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
507
out:
508 509 510
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
511
	btrfs_put_block_group(block_group);
512 513
}

514 515
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     int load_cache_only)
516
{
517
	DEFINE_WAIT(wait);
518 519
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
520 521
	int ret = 0;

522
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
523 524
	if (!caching_ctl)
		return -ENOMEM;
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
	caching_ctl->work.func = caching_thread;

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
565
		return 0;
566 567 568 569 570
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
571

572
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
573 574 575 576
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
577
			cache->caching_ctl = NULL;
578 579 580
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
581 582 583 584 585 586
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
			}
587 588
		}
		spin_unlock(&cache->lock);
589
		wake_up(&caching_ctl->wait);
590
		if (ret == 1) {
591
			put_caching_control(caching_ctl);
592
			free_excluded_extents(fs_info->extent_root, cache);
593
			return 0;
594
		}
595 596 597 598 599 600 601 602 603 604 605 606 607 608
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
609 610
	}

611 612
	if (load_cache_only) {
		put_caching_control(caching_ctl);
613
		return 0;
614 615
	}

616
	down_write(&fs_info->extent_commit_sem);
617
	atomic_inc(&caching_ctl->count);
618 619 620
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

621
	btrfs_get_block_group(cache);
622

623
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
624

625
	return ret;
626 627
}

628 629 630
/*
 * return the block group that starts at or after bytenr
 */
631 632
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
633
{
634
	struct btrfs_block_group_cache *cache;
635

636
	cache = block_group_cache_tree_search(info, bytenr, 0);
637

638
	return cache;
639 640
}

641
/*
642
 * return the block group that contains the given bytenr
643
 */
644 645 646
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
647
{
648
	struct btrfs_block_group_cache *cache;
649

650
	cache = block_group_cache_tree_search(info, bytenr, 1);
651

652
	return cache;
653
}
654

655 656
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
657
{
658 659
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
660

661
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
662

663 664
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
665
		if (found->flags & flags) {
666
			rcu_read_unlock();
667
			return found;
668
		}
669
	}
670
	rcu_read_unlock();
671
	return NULL;
672 673
}

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

689
/* simple helper to search for an existing extent at a given offset */
690
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
691 692 693
{
	int ret;
	struct btrfs_key key;
694
	struct btrfs_path *path;
695

696
	path = btrfs_alloc_path();
697 698 699
	if (!path)
		return -ENOMEM;

700 701
	key.objectid = start;
	key.offset = len;
702
	key.type = BTRFS_EXTENT_ITEM_KEY;
703 704
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
705 706 707 708 709 710
	if (ret > 0) {
		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
		if (key.objectid == start &&
		    key.type == BTRFS_METADATA_ITEM_KEY)
			ret = 0;
	}
711
	btrfs_free_path(path);
712 713 714
	return ret;
}

715
/*
716
 * helper function to lookup reference count and flags of a tree block.
717 718 719 720 721 722 723 724 725
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
726
			     u64 offset, int metadata, u64 *refs, u64 *flags)
727 728 729 730 731 732 733 734 735 736 737 738
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

739 740 741 742 743 744 745 746 747
	/*
	 * If we don't have skinny metadata, don't bother doing anything
	 * different
	 */
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) {
		offset = root->leafsize;
		metadata = 0;
	}

748 749 750 751
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

752 753 754 755 756 757 758 759 760 761
	if (metadata) {
		key.objectid = bytenr;
		key.type = BTRFS_METADATA_ITEM_KEY;
		key.offset = offset;
	} else {
		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = offset;
	}

762 763 764 765 766 767 768 769 770 771
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

772 773 774 775 776 777 778
	if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = root->leafsize;
		btrfs_release_path(path);
		goto again;
	}

779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

818
			btrfs_release_path(path);
819

820 821 822 823
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

849 850 851 852 853 854 855 856 857 858 859 860 861 862
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
908 909 910
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
911
 * - different files inside a single subvolume
912 913
 * - different offsets inside a file (bookend extents in file.c)
 *
914
 * The extent ref structure for the implicit back refs has fields for:
915 916 917
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
918 919
 * - original offset in the file
 * - how many bookend extents
920
 *
921 922
 * The key offset for the implicit back refs is hash of the first
 * three fields.
923
 *
924
 * The extent ref structure for the full back refs has field for:
925
 *
926
 * - number of pointers in the tree leaf
927
 *
928 929
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
930
 *
931 932
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
933
 *
934
 *     (root_key.objectid, inode objectid, offset in file, 1)
935
 *
936 937
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
938
 *
939
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
940
 *
941
 * Btree extents can be referenced by:
942
 *
943
 * - Different subvolumes
944
 *
945 946 947 948
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
949
 *
950 951 952
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
953
 */
954

955 956 957 958 959
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
960
{
961 962 963 964 965
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
966
	struct btrfs_key key;
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
986
				BUG_ON(ret > 0); /* Corruption */
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
1002
	btrfs_release_path(path);
1003 1004 1005 1006 1007 1008 1009 1010 1011

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
1012
	BUG_ON(ret); /* Corruption */
1013

1014
	btrfs_extend_item(root, path, new_size);
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1044
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1045
	lenum = cpu_to_le64(owner);
1046
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1047
	lenum = cpu_to_le64(offset);
1048
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
1081
	struct extent_buffer *leaf;
1082
	u32 nritems;
1083
	int ret;
1084 1085
	int recow;
	int err = -ENOENT;
1086

1087
	key.objectid = bytenr;
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
1103

1104 1105 1106 1107 1108
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1109
		btrfs_release_path(path);
1110 1111 1112 1113 1114 1115 1116 1117 1118
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
1119 1120 1121
	}

	leaf = path->nodes[0];
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1147
				btrfs_release_path(path);
1148 1149 1150 1151 1152 1153
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
1154
	}
1155 1156
fail:
	return err;
1157 1158
}

1159 1160 1161 1162 1163 1164
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
1165 1166 1167
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1168
	u32 size;
1169 1170
	u32 num_refs;
	int ret;
1171 1172

	key.objectid = bytenr;
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1183

1184 1185 1186 1187 1188 1189 1190
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
1191
		ref = btrfs_item_ptr(leaf, path->slots[0],
1192 1193 1194 1195 1196 1197 1198
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1199
		}
1200 1201 1202 1203 1204 1205 1206 1207
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1208
			btrfs_release_path(path);
1209 1210 1211 1212 1213
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
1214

1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1229 1230
		}
	}
1231 1232 1233
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1234
	btrfs_release_path(path);
1235
	return ret;
1236 1237
}

1238 1239 1240 1241
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
1242
{
1243 1244 1245
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
1246
	struct extent_buffer *leaf;
1247
	u32 num_refs = 0;
1248 1249 1250
	int ret = 0;

	leaf = path->nodes[0];
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1272 1273
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1274

1275 1276 1277
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
1290 1291 1292 1293 1294
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1295 1296 1297
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1298
{
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
Chris Mason's avatar
Chris Mason committed
1330
#endif
1331 1332 1333 1334 1335
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1336

1337 1338 1339 1340 1341
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1342
{
1343
	struct btrfs_key key;
1344 1345
	int ret;

1346 1347 1348 1349 1350 1351 1352
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1353 1354
	}

1355 1356 1357 1358 1359
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1360
		btrfs_release_path(path);
1361 1362 1363 1364 1365
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1366
#endif
1367
	return ret;
1368 1369
}

1370 1371 1372 1373 1374
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1375
{
1376
	struct btrfs_key key;
1377 1378
	int ret;

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1389
	btrfs_release_path(path);
1390 1391 1392
	return ret;
}

1393
static inline int extent_ref_type(u64 parent, u64 owner)
1394
{
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
1408
}
1409

1410 1411
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1412

1413
{
1414
	for (; level < BTRFS_MAX_LEVEL; level++) {
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
1430

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1466 1467
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
1468

1469
	key.objectid = bytenr;
1470
	key.type = BTRFS_EXTENT_ITEM_KEY;
1471
	key.offset = num_bytes;
1472

1473 1474 1475
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1476
		path->keep_locks = 1;
1477 1478
	} else
		extra_size = -1;
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489

	/*
	 * Owner is our parent level, so we can just add one to get the level
	 * for the block we are interested in.
	 */
	if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
		key.type = BTRFS_METADATA_ITEM_KEY;
		key.offset = owner;
	}

again:
1490
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1491
	if (ret < 0) {
1492 1493 1494
		err = ret;
		goto out;
	}
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518

	/*
	 * We may be a newly converted file system which still has the old fat
	 * extent entries for metadata, so try and see if we have one of those.
	 */
	if (ret > 0 && skinny_metadata) {
		skinny_metadata = false;
		if (path->slots[0]) {
			path->slots[0]--;
			btrfs_item_key_to_cpu(path->nodes[0], &key,
					      path->slots[0]);
			if (key.objectid == bytenr &&
			    key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes)
				ret = 0;
		}
		if (ret) {
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;
			btrfs_release_path(path);
			goto again;
		}
	}

1519 1520 1521
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
1522 1523 1524 1525
	} else if (ret) {
		err = -EIO;
		WARN_ON(1);
		goto out;
1526
	}
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

1554
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1618 1619
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1620
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1621 1622 1623 1624 1625 1626
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1627
	if (insert) {
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1638
void setup_inline_extent_backref(struct btrfs_root *root,
1639 1640 1641 1642 1643
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1661
	btrfs_extend_item(root, path, size);
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1711
		return ret;
1712

1713
	btrfs_release_path(path);
1714 1715 1716 1717 1718 1719 1720 1721
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1722
	}
1723 1724
	return ret;
}
1725

1726 1727 1728 1729
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1730
void update_inline_extent_backref(struct btrfs_root *root,
1731 1732 1733 1734
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
				  struct btrfs_delayed_extent_op *extent_op)
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1767
	}
1768

1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1786
		btrfs_truncate_item(root, path, item_size, 1);
1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1808
		update_inline_extent_backref(root, path, iref,
1809
					     refs_to_add, extent_op);
1810
	} else if (ret == -ENOENT) {
1811
		setup_inline_extent_backref(root, path, iref, parent,
1812 1813 1814
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1815
	}
1816 1817
	return ret;
}
1818

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1837

1838 1839 1840 1841 1842 1843
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
1844
	int ret = 0;
1845

1846 1847
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1848
		update_inline_extent_backref(root, path, iref,
1849
					     -refs_to_drop, NULL);
1850 1851 1852 1853 1854 1855 1856 1857
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1858
static int btrfs_issue_discard(struct block_device *bdev,
1859 1860
				u64 start, u64 len)
{
1861
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1862 1863 1864
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1865
				u64 num_bytes, u64 *actual_bytes)
1866 1867
{
	int ret;
1868
	u64 discarded_bytes = 0;
1869
	struct btrfs_bio *bbio = NULL;
1870

1871

1872
	/* Tell the block device(s) that the sectors can be discarded */
1873
	ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
1874
			      bytenr, &num_bytes, &bbio, 0);
1875
	/* Error condition is -ENOMEM */
1876
	if (!ret) {
1877
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1878 1879 1880
		int i;


1881
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1882 1883 1884
			if (!stripe->dev->can_discard)
				continue;

1885 1886 1887 1888 1889 1890
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1891
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1892 1893 1894 1895 1896 1897 1898

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1899
		}
1900
		kfree(bbio);
1901
	}
1902 1903 1904 1905

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1906

David Woodhouse's avatar
David Woodhouse committed
1907 1908
	if (ret == -EOPNOTSUPP)
		ret = 0;
1909 1910 1911
	return ret;
}

1912
/* Can return -ENOMEM */
1913 1914 1915
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
1916
			 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1917 1918
{
	int ret;
1919 1920
	struct btrfs_fs_info *fs_info = root->fs_info;

1921 1922 1923 1924
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1925 1926
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1927
					parent, root_objectid, (int)owner,
1928
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1929
	} else {
1930 1931
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1932
					parent, root_objectid, owner, offset,
1933
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1977

1978
	btrfs_mark_buffer_dirty(leaf);
1979
	btrfs_release_path(path);
1980 1981

	path->reada = 1;
1982 1983
	path->leave_spinning = 1;

1984 1985
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1986 1987
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1988 1989
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
1990
out:
1991
	btrfs_free_path(path);
1992
	return err;
1993 1994
}

1995 1996 1997 1998 1999
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2000
{
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2019
		if (extent_op)
2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
			flags |= extent_op->flags_to_set;
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2071
	int ret;
2072
	int err = 0;
2073
	int metadata = !extent_op->is_data;
2074

2075 2076 2077
	if (trans->aborted)
		return 0;

2078 2079 2080
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		metadata = 0;

2081 2082 2083 2084 2085 2086
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;

2087 2088
	if (metadata) {
		key.type = BTRFS_METADATA_ITEM_KEY;
2089
		key.offset = extent_op->level;
2090 2091 2092 2093 2094 2095
	} else {
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = node->num_bytes;
	}

again:
2096 2097 2098 2099 2100 2101 2102 2103 2104
	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
2105 2106 2107 2108 2109 2110 2111 2112
		if (metadata) {
			btrfs_release_path(path);
			metadata = 0;

			key.offset = node->num_bytes;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			goto again;
		}
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2134

2135 2136 2137 2138
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2139 2140
}

2141 2142 2143 2144 2145
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2146 2147
{
	int ret = 0;
2148 2149 2150 2151
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2152 2153
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
2154

2155 2156 2157 2158 2159 2160
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

2161 2162 2163 2164 2165 2166 2167 2168 2169
	ins.objectid = node->bytenr;
	if (skinny_metadata) {
		ins.offset = ref->level;
		ins.type = BTRFS_METADATA_ITEM_KEY;
	} else {
		ins.offset = node->num_bytes;
		ins.type = BTRFS_EXTENT_ITEM_KEY;
	}

2170 2171
	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2172
		BUG_ON(!extent_op || !extent_op->update_flags);
2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2189 2190 2191 2192
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2193 2194 2195 2196 2197
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2198
{
2199 2200 2201 2202 2203
	int ret = 0;

	if (trans->aborted)
		return 0;

2204
	if (btrfs_delayed_ref_is_head(node)) {
2205 2206 2207 2208 2209 2210 2211
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2212 2213
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2214
		if (insert_reserved) {
2215 2216
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2217 2218 2219 2220 2221
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2222
		}
2223
		return ret;
2224 2225
	}

2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2259
		if (ref->action == action)
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2270 2271 2272 2273
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2274 2275 2276
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2277 2278 2279 2280
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2281
	struct btrfs_delayed_extent_op *extent_op;
2282
	struct btrfs_fs_info *fs_info = root->fs_info;
2283
	int ret;
2284
	int count = 0;
2285 2286 2287 2288 2289
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2290 2291
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2292 2293
				break;

2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2311 2312
			}
		}
2313

2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
		/*
		 * We need to try and merge add/drops of the same ref since we
		 * can run into issues with relocate dropping the implicit ref
		 * and then it being added back again before the drop can
		 * finish.  If we merged anything we need to re-loop so we can
		 * get a good ref.
		 */
		btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
					 locked_ref);

2324 2325 2326 2327 2328 2329 2330
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2331
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2332 2333 2334 2335 2336
			/*
			 * there are still refs with lower seq numbers in the
			 * process of being added. Don't run this ref yet.
			 */
			list_del_init(&locked_ref->cluster);
2337
			btrfs_delayed_ref_unlock(locked_ref);
2338 2339 2340 2341 2342 2343 2344 2345
			locked_ref = NULL;
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
			cond_resched();
			spin_lock(&delayed_refs->lock);
			continue;
		}

2346 2347 2348 2349 2350 2351
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2352

2353 2354 2355
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2356 2357 2358 2359 2360 2361
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2362 2363

			if (extent_op && must_insert_reserved) {
2364
				btrfs_free_delayed_extent_op(extent_op);
2365 2366 2367 2368 2369 2370 2371 2372
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
2373
				btrfs_free_delayed_extent_op(extent_op);
2374

2375
				if (ret) {
2376
					btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2377
					spin_lock(&delayed_refs->lock);
2378
					btrfs_delayed_ref_unlock(locked_ref);
2379 2380 2381
					return ret;
				}

2382
				goto next;
2383
			}
2384
		}
2385

2386 2387 2388
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2389
		if (!btrfs_delayed_ref_is_head(ref)) {
2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
			/*
			 * when we play the delayed ref, also correct the
			 * ref_mod on head
			 */
			switch (ref->action) {
			case BTRFS_ADD_DELAYED_REF:
			case BTRFS_ADD_DELAYED_EXTENT:
				locked_ref->node.ref_mod -= ref->ref_mod;
				break;
			case BTRFS_DROP_DELAYED_REF:
				locked_ref->node.ref_mod += ref->ref_mod;
				break;
			default:
				WARN_ON(1);
			}
		}
2406
		spin_unlock(&delayed_refs->lock);
2407

2408
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2409
					  must_insert_reserved);
2410

2411
		btrfs_free_delayed_extent_op(extent_op);
2412
		if (ret) {
2413 2414
			btrfs_delayed_ref_unlock(locked_ref);
			btrfs_put_delayed_ref(ref);
2415
			btrfs_debug(fs_info, "run_one_delayed_ref returned %d", ret);
2416
			spin_lock(&delayed_refs->lock);
2417 2418 2419
			return ret;
		}

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432
		/*
		 * If this node is a head, that means all the refs in this head
		 * have been dealt with, and we will pick the next head to deal
		 * with, so we must unlock the head and drop it from the cluster
		 * list before we release it.
		 */
		if (btrfs_delayed_ref_is_head(ref)) {
			list_del_init(&locked_ref->cluster);
			btrfs_delayed_ref_unlock(locked_ref);
			locked_ref = NULL;
		}
		btrfs_put_delayed_ref(ref);
		count++;
2433
next:
2434 2435 2436 2437 2438 2439
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2483 2484 2485 2486 2487 2488 2489 2490 2491
int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
					 struct btrfs_fs_info *fs_info)
{
	struct qgroup_update *qgroup_update;
	int ret = 0;

	if (list_empty(&trans->qgroup_ref_list) !=
	    !trans->delayed_ref_elem.seq) {
		/* list without seq or seq without list */
2492
		btrfs_err(fs_info,
2493
			"qgroup accounting update error, list is%s empty, seq is %#x.%x",
2494
			list_empty(&trans->qgroup_ref_list) ? "" : " not",
2495 2496
			(u32)(trans->delayed_ref_elem.seq >> 32),
			(u32)trans->delayed_ref_elem.seq);
2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
		BUG();
	}

	if (!trans->delayed_ref_elem.seq)
		return 0;

	while (!list_empty(&trans->qgroup_ref_list)) {
		qgroup_update = list_first_entry(&trans->qgroup_ref_list,
						 struct qgroup_update, list);
		list_del(&qgroup_update->list);
		if (!ret)
			ret = btrfs_qgroup_account_ref(
					trans, fs_info, qgroup_update->node,
					qgroup_update->extent_op);
		kfree(qgroup_update);
	}

	btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);

	return ret;
}

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
static int refs_newer(struct btrfs_delayed_ref_root *delayed_refs, int seq,
		      int count)
{
	int val = atomic_read(&delayed_refs->ref_seq);

	if (val < seq || val >= seq + count)
		return 1;
	return 0;
}

2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573
static inline u64 heads_to_leaves(struct btrfs_root *root, u64 heads)
{
	u64 num_bytes;

	num_bytes = heads * (sizeof(struct btrfs_extent_item) +
			     sizeof(struct btrfs_extent_inline_ref));
	if (!btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		num_bytes += heads * sizeof(struct btrfs_tree_block_info);

	/*
	 * We don't ever fill up leaves all the way so multiply by 2 just to be
	 * closer to what we're really going to want to ouse.
	 */
	return div64_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(root));
}

int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_rsv *global_rsv;
	u64 num_heads = trans->transaction->delayed_refs.num_heads_ready;
	u64 num_bytes;
	int ret = 0;

	num_bytes = btrfs_calc_trans_metadata_size(root, 1);
	num_heads = heads_to_leaves(root, num_heads);
	if (num_heads > 1)
		num_bytes += (num_heads - 1) * root->leafsize;
	num_bytes <<= 1;
	global_rsv = &root->fs_info->global_block_rsv;

	/*
	 * If we can't allocate any more chunks lets make sure we have _lots_ of
	 * wiggle room since running delayed refs can create more delayed refs.
	 */
	if (global_rsv->space_info->full)
		num_bytes <<= 1;

	spin_lock(&global_rsv->lock);
	if (global_rsv->reserved <= num_bytes)
		ret = 1;
	spin_unlock(&global_rsv->lock);
	return ret;
}

2574 2575 2576 2577 2578 2579
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2580 2581 2582
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2583 2584 2585 2586 2587 2588 2589 2590 2591
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
2592
	u64 delayed_start;
2593 2594
	int run_all = count == (unsigned long)-1;
	int run_most = 0;
2595
	int loops;
2596

2597 2598 2599 2600
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2601 2602 2603
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

2604 2605
	btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);

2606 2607
	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}

	if (!run_all && !run_most) {
		int old;
		int seq = atomic_read(&delayed_refs->ref_seq);

progress:
		old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
		if (old) {
			DEFINE_WAIT(__wait);
2621 2622
			if (delayed_refs->flushing ||
			    !btrfs_should_throttle_delayed_refs(trans, root))
2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646
				return 0;

			prepare_to_wait(&delayed_refs->wait, &__wait,
					TASK_UNINTERRUPTIBLE);

			old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
			if (old) {
				schedule();
				finish_wait(&delayed_refs->wait, &__wait);

				if (!refs_newer(delayed_refs, seq, 256))
					goto progress;
				else
					return 0;
			} else {
				finish_wait(&delayed_refs->wait, &__wait);
				goto again;
			}
		}

	} else {
		atomic_inc(&delayed_refs->procs_running_refs);
	}

2647
again:
2648
	loops = 0;
2649
	spin_lock(&delayed_refs->lock);
2650

2651 2652 2653 2654
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif

2655 2656
	while (1) {
		if (!(run_all || run_most) &&
2657
		    !btrfs_should_throttle_delayed_refs(trans, root))
2658
			break;
2659

2660
		/*
2661 2662 2663 2664
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2665
		 */
2666
		delayed_start = delayed_refs->run_delayed_start;
2667 2668 2669
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2670 2671
			break;

2672
		ret = run_clustered_refs(trans, root, &cluster);
2673
		if (ret < 0) {
2674
			btrfs_release_ref_cluster(&cluster);
2675 2676
			spin_unlock(&delayed_refs->lock);
			btrfs_abort_transaction(trans, root, ret);
2677
			atomic_dec(&delayed_refs->procs_running_refs);
2678
			wake_up(&delayed_refs->wait);
2679 2680
			return ret;
		}
2681

2682 2683
		atomic_add(ret, &delayed_refs->ref_seq);

2684 2685 2686 2687
		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2688

2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706
		if (delayed_start >= delayed_refs->run_delayed_start) {
			if (loops == 0) {
				/*
				 * btrfs_find_ref_cluster looped. let's do one
				 * more cycle. if we don't run any delayed ref
				 * during that cycle (because we can't because
				 * all of them are blocked), bail out.
				 */
				loops = 1;
			} else {
				/*
				 * no runnable refs left, stop trying
				 */
				BUG_ON(run_all);
				break;
			}
		}
		if (ret) {
2707
			/* refs were run, let's reset staleness detection */
2708
			loops = 0;
2709
		}
2710
	}
2711

2712
	if (run_all) {
2713 2714 2715 2716 2717 2718
		if (!list_empty(&trans->new_bgs)) {
			spin_unlock(&delayed_refs->lock);
			btrfs_create_pending_block_groups(trans, root);
			spin_lock(&delayed_refs->lock);
		}

2719
		node = rb_first(&delayed_refs->root);
2720
		if (!node)
2721
			goto out;
2722
		count = (unsigned long)-1;
2723

2724 2725 2726 2727 2728
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2729

2730 2731 2732 2733
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2734 2735 2736 2737
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2738 2739 2740 2741
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2742
				cond_resched();
2743 2744 2745 2746 2747 2748 2749
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2750
	}
2751
out:
2752 2753 2754 2755 2756
	atomic_dec(&delayed_refs->procs_running_refs);
	smp_mb();
	if (waitqueue_active(&delayed_refs->wait))
		wake_up(&delayed_refs->wait);

2757
	spin_unlock(&delayed_refs->lock);
2758
	assert_qgroups_uptodate(trans);
2759 2760 2761
	return 0;
}

2762 2763 2764
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
2765
				int level, int is_data)
2766 2767 2768 2769
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

2770
	extent_op = btrfs_alloc_delayed_extent_op();
2771 2772 2773 2774 2775 2776 2777
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;
2778
	extent_op->level = level;
2779

2780 2781
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2782
	if (ret)
2783
		btrfs_free_delayed_extent_op(extent_op);
2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2810
		btrfs_release_path(path);
2811

2812 2813 2814 2815
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
2839 2840
		int seq = ref->seq;

2841
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2842
		if (ref->bytenr == bytenr && ref->seq == seq)
2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2862 2863
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2864
	struct extent_buffer *leaf;
2865 2866 2867
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2868
	struct btrfs_key key;
2869
	u32 item_size;
2870
	int ret;
2871

2872
	key.objectid = bytenr;
2873
	key.offset = (u64)-1;
2874
	key.type = BTRFS_EXTENT_ITEM_KEY;
2875 2876 2877 2878

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2879
	BUG_ON(ret == 0); /* Corruption */
Yan Zheng's avatar
Yan Zheng committed
2880 2881 2882

	ret = -ENOENT;
	if (path->slots[0] == 0)
2883
		goto out;
2884

2885
	path->slots[0]--;
2886
	leaf = path->nodes[0];
2887
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2888

2889
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2890
		goto out;
2891

2892 2893 2894 2895 2896 2897 2898 2899 2900
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2901

2902 2903 2904
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2905

2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2945
			goto out;
Yan Zheng's avatar
Yan Zheng committed
2946

2947 2948 2949 2950 2951 2952 2953
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2954
	}
2955 2956 2957

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2958
out:
Yan Zheng's avatar
Yan Zheng committed
2959
	btrfs_free_path(path);
2960 2961
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2962
	return ret;
2963
}
Chris Mason's avatar
Chris Mason committed
2964

2965
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2966
			   struct btrfs_root *root,
2967
			   struct extent_buffer *buf,
2968
			   int full_backref, int inc, int for_cow)
2969 2970
{
	u64 bytenr;
2971 2972
	u64 num_bytes;
	u64 parent;
2973 2974 2975 2976 2977 2978 2979 2980
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2981
			    u64, u64, u64, u64, u64, u64, int);
2982 2983 2984 2985 2986

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2987 2988
	if (!root->ref_cows && level == 0)
		return 0;
2989

2990 2991 2992 2993
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
2994

2995 2996 2997 2998 2999 3000
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
3001
		if (level == 0) {
3002
			btrfs_item_key_to_cpu(buf, &key, i);
3003 3004
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
3005
			fi = btrfs_item_ptr(buf, i,
3006 3007 3008 3009 3010 3011 3012
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
3013 3014 3015 3016 3017

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
3018
					   key.offset, for_cow);
3019 3020 3021
			if (ret)
				goto fail;
		} else {
3022 3023 3024
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
3025 3026
					   parent, ref_root, level - 1, 0,
					   for_cow);
3027 3028 3029 3030 3031 3032
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
3033 3034 3035 3036
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3037
		  struct extent_buffer *buf, int full_backref, int for_cow)
3038
{
3039
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
3040 3041 3042
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3043
		  struct extent_buffer *buf, int full_backref, int for_cow)
3044
{
3045
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
3046 3047
}

3048 3049 3050 3051 3052 3053 3054
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
3055 3056
	unsigned long bi;
	struct extent_buffer *leaf;
3057 3058

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
3059 3060
	if (ret < 0)
		goto fail;
3061
	BUG_ON(ret); /* Corruption */
3062 3063 3064 3065 3066

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
3067
	btrfs_release_path(path);
3068
fail:
3069 3070
	if (ret) {
		btrfs_abort_transaction(trans, root, ret);
3071
		return ret;
3072
	}
3073 3074 3075 3076
	return 0;

}

3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
3088
		btrfs_get_block_group(cache);
3089 3090 3091 3092 3093 3094
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

3095 3096 3097 3098 3099 3100 3101
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
3102
	int dcs = BTRFS_DC_ERROR;
3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
3122
		btrfs_release_path(path);
3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

3139 3140 3141 3142 3143 3144 3145
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
3156 3157 3158 3159 3160
		ret = btrfs_check_trunc_cache_free_space(root,
					&root->fs_info->global_block_rsv);
		if (ret)
			goto out_put;

3161 3162 3163 3164 3165 3166 3167
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
3168 3169 3170 3171 3172 3173 3174
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
	    !btrfs_test_opt(root, SPACE_CACHE)) {
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
3175
		dcs = BTRFS_DC_WRITTEN;
3176 3177 3178 3179 3180
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

3181 3182 3183 3184 3185 3186 3187
	/*
	 * Try to preallocate enough space based on how big the block group is.
	 * Keep in mind this has to include any pinned space which could end up
	 * taking up quite a bit since it's not folded into the other space
	 * cache.
	 */
	num_pages = (int)div64_u64(block_group->key.offset, 256 * 1024 * 1024);
3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200
	if (!num_pages)
		num_pages = 1;

	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3201 3202
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3203
	btrfs_free_reserved_data_space(inode, num_pages);
3204

3205 3206 3207
out_put:
	iput(inode);
out_free:
3208
	btrfs_release_path(path);
3209 3210
out:
	spin_lock(&block_group->lock);
3211
	if (!ret && dcs == BTRFS_DC_SETUP)
3212
		block_group->cache_generation = trans->transid;
3213
	block_group->disk_cache_state = dcs;
3214 3215 3216 3217 3218
	spin_unlock(&block_group->lock);

	return ret;
}

3219 3220
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
3221
{
3222
	struct btrfs_block_group_cache *cache;
3223 3224
	int err = 0;
	struct btrfs_path *path;
3225
	u64 last = 0;
3226 3227 3228 3229 3230

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

3250
	while (1) {
3251 3252 3253
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3254 3255
			if (err) /* File system offline */
				goto out;
3256
		}
3257

3258 3259
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
3260 3261 3262 3263 3264
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

3265 3266 3267 3268 3269 3270 3271 3272 3273 3274
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
3275

3276 3277
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3278
		cache->dirty = 0;
3279
		last = cache->key.objectid + cache->key.offset;
3280

3281
		err = write_one_cache_group(trans, root, path, cache);
3282 3283 3284
		if (err) /* File system offline */
			goto out;

3285
		btrfs_put_block_group(cache);
3286
	}
3287

3288 3289 3290 3291 3292 3293 3294 3295 3296
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3297 3298
			if (err) /* File system offline */
				goto out;
3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

3323
		err = btrfs_write_out_cache(root, trans, cache, path);
3324 3325 3326 3327 3328

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
3329
		if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
3330 3331 3332 3333
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}
3334
out:
3335

3336
	btrfs_free_path(path);
3337
	return err;
3338 3339
}

3340 3341 3342 3343 3344 3345 3346 3347 3348
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3349
		btrfs_put_block_group(block_group);
3350 3351 3352
	return readonly;
}

3353 3354 3355 3356 3357
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3358 3359 3360 3361 3362 3363 3364 3365
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3366 3367 3368

	found = __find_space_info(info, flags);
	if (found) {
3369
		spin_lock(&found->lock);
3370
		found->total_bytes += total_bytes;
Josef Bacik's avatar
Josef Bacik committed
3371
		found->disk_total += total_bytes * factor;
3372
		found->bytes_used += bytes_used;
3373
		found->disk_used += bytes_used * factor;
3374
		found->full = 0;
3375
		spin_unlock(&found->lock);
3376 3377 3378
		*space_info = found;
		return 0;
	}
3379
	found = kzalloc(sizeof(*found), GFP_NOFS);
3380 3381 3382
	if (!found)
		return -ENOMEM;

3383 3384
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3385
	init_rwsem(&found->groups_sem);
3386
	spin_lock_init(&found->lock);
3387
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3388
	found->total_bytes = total_bytes;
Josef Bacik's avatar
Josef Bacik committed
3389
	found->disk_total = total_bytes * factor;
3390
	found->bytes_used = bytes_used;
3391
	found->disk_used = bytes_used * factor;
3392
	found->bytes_pinned = 0;
3393
	found->bytes_reserved = 0;
3394
	found->bytes_readonly = 0;
3395
	found->bytes_may_use = 0;
3396
	found->full = 0;
3397
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3398
	found->chunk_alloc = 0;
3399 3400
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3401
	*space_info = found;
3402
	list_add_rcu(&found->list, &info->space_info);
3403 3404
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
3405 3406 3407
	return 0;
}

3408 3409
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3410 3411
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3412

3413
	write_seqlock(&fs_info->profiles_lock);
3414 3415 3416 3417 3418 3419
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3420
	write_sequnlock(&fs_info->profiles_lock);
3421
}
3422

3423 3424 3425
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3426 3427
 *
 * should be called with either volume_mutex or balance_lock held
3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3451 3452 3453
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3454 3455 3456
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3457
 */
3458
static u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3459
{
3460 3461 3462 3463 3464 3465 3466
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3467
	u64 target;
David Woodhouse's avatar
David Woodhouse committed
3468
	u64 tmp;
3469

3470 3471 3472 3473
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3474
	spin_lock(&root->fs_info->balance_lock);
3475 3476 3477 3478
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3479
			spin_unlock(&root->fs_info->balance_lock);
3480
			return extended_to_chunk(target);
3481 3482 3483 3484
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

David Woodhouse's avatar
David Woodhouse committed
3485
	/* First, mask out the RAID levels which aren't possible */
3486
	if (num_devices == 1)
David Woodhouse's avatar
David Woodhouse committed
3487 3488 3489 3490
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0 |
			   BTRFS_BLOCK_GROUP_RAID5);
	if (num_devices < 3)
		flags &= ~BTRFS_BLOCK_GROUP_RAID6;
3491 3492 3493
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

David Woodhouse's avatar
David Woodhouse committed
3494 3495 3496 3497
	tmp = flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID0 |
		       BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID5 |
		       BTRFS_BLOCK_GROUP_RAID6 | BTRFS_BLOCK_GROUP_RAID10);
	flags &= ~tmp;
3498

David Woodhouse's avatar
David Woodhouse committed
3499 3500 3501 3502 3503 3504 3505 3506 3507 3508
	if (tmp & BTRFS_BLOCK_GROUP_RAID6)
		tmp = BTRFS_BLOCK_GROUP_RAID6;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID5)
		tmp = BTRFS_BLOCK_GROUP_RAID5;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID10)
		tmp = BTRFS_BLOCK_GROUP_RAID10;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID1)
		tmp = BTRFS_BLOCK_GROUP_RAID1;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID0)
		tmp = BTRFS_BLOCK_GROUP_RAID0;
3509

David Woodhouse's avatar
David Woodhouse committed
3510
	return extended_to_chunk(flags | tmp);
3511 3512
}

3513
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
3514
{
3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526
	unsigned seq;

	do {
		seq = read_seqbegin(&root->fs_info->profiles_lock);

		if (flags & BTRFS_BLOCK_GROUP_DATA)
			flags |= root->fs_info->avail_data_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			flags |= root->fs_info->avail_system_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_METADATA)
			flags |= root->fs_info->avail_metadata_alloc_bits;
	} while (read_seqretry(&root->fs_info->profiles_lock, seq));
3527

3528
	return btrfs_reduce_alloc_profile(root, flags);
3529 3530
}

3531
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
Josef Bacik's avatar
Josef Bacik committed
3532
{
3533
	u64 flags;
David Woodhouse's avatar
David Woodhouse committed
3534
	u64 ret;
Josef Bacik's avatar
Josef Bacik committed
3535

3536 3537 3538 3539
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
Josef Bacik's avatar
Josef Bacik committed
3540
	else
3541
		flags = BTRFS_BLOCK_GROUP_METADATA;
Josef Bacik's avatar
Josef Bacik committed
3542

David Woodhouse's avatar
David Woodhouse committed
3543 3544
	ret = get_alloc_profile(root, flags);
	return ret;
3545
}
Josef Bacik's avatar
Josef Bacik committed
3546

3547 3548 3549 3550
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3551
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
3552 3553
{
	struct btrfs_space_info *data_sinfo;
3554
	struct btrfs_root *root = BTRFS_I(inode)->root;
3555
	struct btrfs_fs_info *fs_info = root->fs_info;
3556
	u64 used;
3557
	int ret = 0, committed = 0, alloc_chunk = 1;
3558 3559

	/* make sure bytes are sectorsize aligned */
3560
	bytes = ALIGN(bytes, root->sectorsize);
3561

3562 3563
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3564 3565 3566 3567
		alloc_chunk = 0;
		committed = 1;
	}

3568
	data_sinfo = fs_info->data_sinfo;
3569 3570
	if (!data_sinfo)
		goto alloc;
Josef Bacik's avatar
Josef Bacik committed
3571

3572 3573 3574
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3575 3576 3577
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3578 3579

	if (used + bytes > data_sinfo->total_bytes) {
3580
		struct btrfs_trans_handle *trans;
Josef Bacik's avatar
Josef Bacik committed
3581

3582 3583 3584 3585
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3586
		if (!data_sinfo->full && alloc_chunk) {
3587
			u64 alloc_target;
Josef Bacik's avatar
Josef Bacik committed
3588

3589
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
3590
			spin_unlock(&data_sinfo->lock);
3591
alloc:
3592
			alloc_target = btrfs_get_alloc_profile(root, 1);
3593
			trans = btrfs_join_transaction(root);
3594 3595
			if (IS_ERR(trans))
				return PTR_ERR(trans);
Josef Bacik's avatar
Josef Bacik committed
3596

3597
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3598 3599
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
3600
			btrfs_end_transaction(trans, root);
3601 3602 3603 3604 3605 3606
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
Josef Bacik's avatar
Josef Bacik committed
3607

3608 3609 3610
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

3611 3612
			goto again;
		}
3613 3614 3615 3616 3617 3618 3619

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
3620 3621
		spin_unlock(&data_sinfo->lock);

3622
		/* commit the current transaction and try again */
3623
commit_trans:
Josef Bacik's avatar
Josef Bacik committed
3624 3625
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3626
			committed = 1;
3627
			trans = btrfs_join_transaction(root);
3628 3629
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3630 3631 3632 3633 3634
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
Josef Bacik's avatar
Josef Bacik committed
3635

3636 3637 3638
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
Josef Bacik's avatar
Josef Bacik committed
3639
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3640
				      data_sinfo->flags, bytes, 1);
3641 3642
	spin_unlock(&data_sinfo->lock);

Josef Bacik's avatar
Josef Bacik committed
3643 3644
	return 0;
}
3645 3646

/*
3647
 * Called if we need to clear a data reservation for this inode.
3648
 */
3649
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3650
{
3651
	struct btrfs_root *root = BTRFS_I(inode)->root;
3652
	struct btrfs_space_info *data_sinfo;
3653

3654
	/* make sure bytes are sectorsize aligned */
3655
	bytes = ALIGN(bytes, root->sectorsize);
3656

3657
	data_sinfo = root->fs_info->data_sinfo;
3658 3659
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
Josef Bacik's avatar
Josef Bacik committed
3660
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3661
				      data_sinfo->flags, bytes, 0);
3662
	spin_unlock(&data_sinfo->lock);
3663 3664
}

3665
static void force_metadata_allocation(struct btrfs_fs_info *info)
3666
{
3667 3668
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3669

3670 3671 3672
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3673
			found->force_alloc = CHUNK_ALLOC_FORCE;
3674
	}
3675
	rcu_read_unlock();
3676 3677
}

3678 3679 3680 3681 3682
static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
{
	return (global->size << 1);
}

3683
static int should_alloc_chunk(struct btrfs_root *root,
3684
			      struct btrfs_space_info *sinfo, int force)
3685
{
3686
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3687
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3688
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3689
	u64 thresh;
3690

3691 3692 3693
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3694 3695 3696 3697 3698
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
3699
	if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
3700
		num_allocated += calc_global_rsv_need_space(global_rsv);
3701

3702 3703 3704 3705 3706
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3707
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3708 3709 3710 3711 3712 3713 3714
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

3715
	if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3716
		return 0;
3717
	return 1;
3718 3719
}

3720 3721 3722 3723
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

David Woodhouse's avatar
David Woodhouse committed
3724 3725 3726 3727
	if (type & (BTRFS_BLOCK_GROUP_RAID10 |
		    BTRFS_BLOCK_GROUP_RAID0 |
		    BTRFS_BLOCK_GROUP_RAID5 |
		    BTRFS_BLOCK_GROUP_RAID6))
3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
3753 3754
		btrfs_info(root->fs_info, "left=%llu, need=%llu, flags=%llu",
			left, thresh, type);
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3766
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3767
			  struct btrfs_root *extent_root, u64 flags, int force)
Josef Bacik's avatar
Josef Bacik committed
3768
{
3769
	struct btrfs_space_info *space_info;
3770
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3771
	int wait_for_alloc = 0;
Josef Bacik's avatar
Josef Bacik committed
3772 3773
	int ret = 0;

3774 3775 3776 3777
	/* Don't re-enter if we're already allocating a chunk */
	if (trans->allocating_chunk)
		return -ENOSPC;

3778
	space_info = __find_space_info(extent_root->fs_info, flags);
3779 3780 3781
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3782
		BUG_ON(ret); /* -ENOMEM */
Josef Bacik's avatar
Josef Bacik committed
3783
	}
3784
	BUG_ON(!space_info); /* Logic error */
Josef Bacik's avatar
Josef Bacik committed
3785

3786
again:
3787
	spin_lock(&space_info->lock);
3788
	if (force < space_info->force_alloc)
3789
		force = space_info->force_alloc;
3790 3791
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3792
		return 0;
Josef Bacik's avatar
Josef Bacik committed
3793 3794
	}

3795
	if (!should_alloc_chunk(extent_root, space_info, force)) {
3796
		spin_unlock(&space_info->lock);
3797 3798 3799 3800 3801
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
Josef Bacik's avatar
Josef Bacik committed
3802
	}
3803

3804
	spin_unlock(&space_info->lock);
Josef Bacik's avatar
Josef Bacik committed
3805

3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3820 3821
	trans->allocating_chunk = true;

3822 3823 3824 3825 3826 3827 3828
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3829 3830 3831 3832 3833
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
Josef Bacik's avatar
Josef Bacik committed
3834
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3835 3836 3837 3838
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
Josef Bacik's avatar
Josef Bacik committed
3839 3840
	}

3841 3842 3843 3844 3845 3846
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Yan Zheng's avatar
Yan Zheng committed
3847
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3848
	trans->allocating_chunk = false;
3849

Josef Bacik's avatar
Josef Bacik committed
3850
	spin_lock(&space_info->lock);
3851 3852
	if (ret < 0 && ret != -ENOSPC)
		goto out;
Josef Bacik's avatar
Josef Bacik committed
3853
	if (ret)
3854
		space_info->full = 1;
3855 3856
	else
		ret = 1;
3857

3858
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3859
out:
3860
	space_info->chunk_alloc = 0;
Josef Bacik's avatar
Josef Bacik committed
3861
	spin_unlock(&space_info->lock);
3862
	mutex_unlock(&fs_info->chunk_mutex);
3863
	return ret;
3864
}
Josef Bacik's avatar
Josef Bacik committed
3865

Josef Bacik's avatar
Josef Bacik committed
3866 3867
static int can_overcommit(struct btrfs_root *root,
			  struct btrfs_space_info *space_info, u64 bytes,
3868
			  enum btrfs_reserve_flush_enum flush)
Josef Bacik's avatar
Josef Bacik committed
3869
{
3870
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
Josef Bacik's avatar
Josef Bacik committed
3871
	u64 profile = btrfs_get_alloc_profile(root, 0);
3872
	u64 space_size;
Josef Bacik's avatar
Josef Bacik committed
3873 3874
	u64 avail;
	u64 used;
3875
	u64 to_add;
Josef Bacik's avatar
Josef Bacik committed
3876 3877

	used = space_info->bytes_used + space_info->bytes_reserved +
3878 3879 3880 3881 3882 3883 3884 3885
		space_info->bytes_pinned + space_info->bytes_readonly;

	/*
	 * We only want to allow over committing if we have lots of actual space
	 * free, but if we don't have enough space to handle the global reserve
	 * space then we could end up having a real enospc problem when trying
	 * to allocate a chunk or some other such important allocation.
	 */
3886 3887 3888 3889
	spin_lock(&global_rsv->lock);
	space_size = calc_global_rsv_need_space(global_rsv);
	spin_unlock(&global_rsv->lock);
	if (used + space_size >= space_info->total_bytes)
3890 3891 3892
		return 0;

	used += space_info->bytes_may_use;
Josef Bacik's avatar
Josef Bacik committed
3893 3894 3895 3896 3897 3898 3899

	spin_lock(&root->fs_info->free_chunk_lock);
	avail = root->fs_info->free_chunk_space;
	spin_unlock(&root->fs_info->free_chunk_lock);

	/*
	 * If we have dup, raid1 or raid10 then only half of the free
David Woodhouse's avatar
David Woodhouse committed
3900 3901 3902
	 * space is actually useable.  For raid56, the space info used
	 * doesn't include the parity drive, so we don't have to
	 * change the math
Josef Bacik's avatar
Josef Bacik committed
3903 3904 3905 3906 3907 3908
	 */
	if (profile & (BTRFS_BLOCK_GROUP_DUP |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_RAID10))
		avail >>= 1;

3909 3910
	to_add = space_info->total_bytes;

Josef Bacik's avatar
Josef Bacik committed
3911
	/*
3912 3913 3914
	 * If we aren't flushing all things, let us overcommit up to
	 * 1/2th of the space. If we can flush, don't let us overcommit
	 * too much, let it overcommit up to 1/8 of the space.
Josef Bacik's avatar
Josef Bacik committed
3915
	 */
3916
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
3917
		to_add >>= 3;
Josef Bacik's avatar
Josef Bacik committed
3918
	else
3919
		to_add >>= 1;
Josef Bacik's avatar
Josef Bacik committed
3920

3921 3922 3923 3924 3925
	/*
	 * Limit the overcommit to the amount of free space we could possibly
	 * allocate for chunks.
	 */
	to_add = min(avail, to_add);
Josef Bacik's avatar
Josef Bacik committed
3926

3927
	if (used + bytes < space_info->total_bytes + to_add)
Josef Bacik's avatar
Josef Bacik committed
3928 3929 3930 3931
		return 1;
	return 0;
}

3932 3933
static void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root,
					 unsigned long nr_pages)
3934 3935 3936 3937 3938
{
	struct super_block *sb = root->fs_info->sb;
	int started;

	/* If we can not start writeback, just sync all the delalloc file. */
3939
	started = try_to_writeback_inodes_sb_nr(sb, nr_pages,
3940 3941 3942 3943 3944 3945 3946 3947 3948
						      WB_REASON_FS_FREE_SPACE);
	if (!started) {
		/*
		 * We needn't worry the filesystem going from r/w to r/o though
		 * we don't acquire ->s_umount mutex, because the filesystem
		 * should guarantee the delalloc inodes list be empty after
		 * the filesystem is readonly(all dirty pages are written to
		 * the disk).
		 */
3949
		btrfs_start_all_delalloc_inodes(root->fs_info, 0);
3950
		if (!current->journal_info)
3951
			btrfs_wait_all_ordered_extents(root->fs_info, 0);
3952 3953 3954
	}
}

Josef Bacik's avatar
Josef Bacik committed
3955
/*
3956
 * shrink metadata reservation for delalloc
Josef Bacik's avatar
Josef Bacik committed
3957
 */
Josef Bacik's avatar
Josef Bacik committed
3958 3959
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
3960
{
3961
	struct btrfs_block_rsv *block_rsv;
3962
	struct btrfs_space_info *space_info;
3963
	struct btrfs_trans_handle *trans;
Josef Bacik's avatar
Josef Bacik committed
3964
	u64 delalloc_bytes;
3965
	u64 max_reclaim;
3966
	long time_left;
3967
	unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3968
	int loops = 0;
3969
	enum btrfs_reserve_flush_enum flush;
3970

3971
	trans = (struct btrfs_trans_handle *)current->journal_info;
3972
	block_rsv = &root->fs_info->delalloc_block_rsv;
3973
	space_info = block_rsv->space_info;
3974 3975

	smp_mb();
3976 3977
	delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
Josef Bacik's avatar
Josef Bacik committed
3978
	if (delalloc_bytes == 0) {
3979
		if (trans)
Josef Bacik's avatar
Josef Bacik committed
3980
			return;
3981
		btrfs_wait_all_ordered_extents(root->fs_info, 0);
Josef Bacik's avatar
Josef Bacik committed
3982
		return;
3983 3984
	}

Josef Bacik's avatar
Josef Bacik committed
3985 3986 3987
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
		nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
3988
		btrfs_writeback_inodes_sb_nr(root, nr_pages);
3989 3990 3991 3992 3993 3994 3995
		/*
		 * We need to wait for the async pages to actually start before
		 * we do anything.
		 */
		wait_event(root->fs_info->async_submit_wait,
			   !atomic_read(&root->fs_info->async_delalloc_pages));

3996 3997 3998 3999
		if (!trans)
			flush = BTRFS_RESERVE_FLUSH_ALL;
		else
			flush = BTRFS_RESERVE_NO_FLUSH;
4000
		spin_lock(&space_info->lock);
4001
		if (can_overcommit(root, space_info, orig, flush)) {
Josef Bacik's avatar
Josef Bacik committed
4002 4003 4004
			spin_unlock(&space_info->lock);
			break;
		}
4005
		spin_unlock(&space_info->lock);
4006

4007
		loops++;
4008
		if (wait_ordered && !trans) {
4009
			btrfs_wait_all_ordered_extents(root->fs_info, 0);
4010
		} else {
Josef Bacik's avatar
Josef Bacik committed
4011
			time_left = schedule_timeout_killable(1);
4012 4013 4014
			if (time_left)
				break;
		}
Josef Bacik's avatar
Josef Bacik committed
4015
		smp_mb();
4016 4017
		delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
4018 4019 4020
	}
}

4021 4022 4023 4024 4025
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
4026
 *
4027 4028 4029
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
4030
 */
4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
	spin_lock(&space_info->lock);
	if (space_info->bytes_pinned >= bytes) {
		spin_unlock(&space_info->lock);
		goto commit;
	}
	spin_unlock(&space_info->lock);

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

4060
	spin_lock(&space_info->lock);
4061
	spin_lock(&delayed_rsv->lock);
4062
	if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
4063
		spin_unlock(&delayed_rsv->lock);
4064
		spin_unlock(&space_info->lock);
4065 4066 4067
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);
4068
	spin_unlock(&space_info->lock);
4069 4070 4071 4072 4073 4074 4075 4076 4077

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

4078
enum flush_state {
4079 4080 4081 4082
	FLUSH_DELAYED_ITEMS_NR	=	1,
	FLUSH_DELAYED_ITEMS	=	2,
	FLUSH_DELALLOC		=	3,
	FLUSH_DELALLOC_WAIT	=	4,
4083 4084
	ALLOC_CHUNK		=	5,
	COMMIT_TRANS		=	6,
4085 4086 4087 4088 4089 4090 4091 4092
};

static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
Josef Bacik's avatar
Josef Bacik committed
4093
	int ret = 0;
4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115

	switch (state) {
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
		if (state == FLUSH_DELAYED_ITEMS_NR) {
			u64 bytes = btrfs_calc_trans_metadata_size(root, 1);

			nr = (int)div64_u64(num_bytes, bytes);
			if (!nr)
				nr = 1;
			nr *= 2;
		} else {
			nr = -1;
		}
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
4116 4117 4118 4119 4120
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
		shrink_delalloc(root, num_bytes, orig_bytes,
				state == FLUSH_DELALLOC_WAIT);
		break;
4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133
	case ALLOC_CHUNK:
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
				     btrfs_get_alloc_profile(root, 0),
				     CHUNK_ALLOC_NO_FORCE);
		btrfs_end_transaction(trans, root);
		if (ret == -ENOSPC)
			ret = 0;
		break;
4134 4135 4136 4137 4138 4139 4140 4141 4142 4143
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
4144 4145 4146 4147 4148
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
4149
 * @flush - whether or not we can flush to make our reservation
4150
 *
4151 4152 4153 4154 4155 4156
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
4157
 */
4158
static int reserve_metadata_bytes(struct btrfs_root *root,
4159
				  struct btrfs_block_rsv *block_rsv,
4160 4161
				  u64 orig_bytes,
				  enum btrfs_reserve_flush_enum flush)
Josef Bacik's avatar
Josef Bacik committed
4162
{
4163
	struct btrfs_space_info *space_info = block_rsv->space_info;
4164
	u64 used;
4165
	u64 num_bytes = orig_bytes;
4166
	int flush_state = FLUSH_DELAYED_ITEMS_NR;
4167
	int ret = 0;
4168
	bool flushing = false;
Josef Bacik's avatar
Josef Bacik committed
4169

4170
again:
4171
	ret = 0;
4172
	spin_lock(&space_info->lock);
4173
	/*
4174 4175
	 * We only want to wait if somebody other than us is flushing and we
	 * are actually allowed to flush all things.
4176
	 */
4177 4178
	while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
	       space_info->flush) {
4179 4180 4181 4182 4183 4184 4185
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
4186
		if (current->journal_info)
4187
			return -EAGAIN;
Arne Jansen's avatar
Arne Jansen committed
4188 4189 4190
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
4191 4192 4193 4194 4195 4196
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
4197 4198 4199
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
Josef Bacik's avatar
Josef Bacik committed
4200

4201 4202 4203 4204 4205 4206 4207
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
4208 4209
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
4210
			space_info->bytes_may_use += orig_bytes;
Josef Bacik's avatar
Josef Bacik committed
4211
			trace_btrfs_space_reservation(root->fs_info,
4212
				"space_info", space_info->flags, orig_bytes, 1);
4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
4228
		num_bytes = used - space_info->total_bytes +
4229
			(orig_bytes * 2);
4230
	}
Josef Bacik's avatar
Josef Bacik committed
4231

4232 4233 4234 4235 4236 4237
	if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
4238 4239
	}

4240 4241 4242 4243
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
4244 4245 4246
	 *
	 * We make the other tasks wait for the flush only when we can flush
	 * all things.
4247
	 */
4248
	if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
4249 4250
		flushing = true;
		space_info->flush = 1;
4251
	}
Josef Bacik's avatar
Josef Bacik committed
4252

4253
	spin_unlock(&space_info->lock);
Josef Bacik's avatar
Josef Bacik committed
4254

4255
	if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
4256
		goto out;
4257

4258 4259 4260
	ret = flush_space(root, space_info, num_bytes, orig_bytes,
			  flush_state);
	flush_state++;
4261 4262 4263 4264 4265 4266 4267 4268 4269 4270

	/*
	 * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock
	 * would happen. So skip delalloc flush.
	 */
	if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
	    (flush_state == FLUSH_DELALLOC ||
	     flush_state == FLUSH_DELALLOC_WAIT))
		flush_state = ALLOC_CHUNK;

4271
	if (!ret)
4272
		goto again;
4273 4274 4275 4276 4277
	else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
		 flush_state < COMMIT_TRANS)
		goto again;
	else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
		 flush_state <= COMMIT_TRANS)
4278 4279 4280
		goto again;

out:
4281 4282 4283 4284 4285 4286 4287 4288 4289
	if (ret == -ENOSPC &&
	    unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
		struct btrfs_block_rsv *global_rsv =
			&root->fs_info->global_block_rsv;

		if (block_rsv != global_rsv &&
		    !block_rsv_use_bytes(global_rsv, orig_bytes))
			ret = 0;
	}
4290
	if (flushing) {
4291
		spin_lock(&space_info->lock);
4292 4293
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
4294
		spin_unlock(&space_info->lock);
4295 4296 4297 4298
	}
	return ret;
}

4299 4300 4301
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
4302
{
4303 4304
	struct btrfs_block_rsv *block_rsv = NULL;

4305 4306 4307 4308
	if (root->ref_cows)
		block_rsv = trans->block_rsv;

	if (root == root->fs_info->csum_root && trans->adding_csums)
4309
		block_rsv = trans->block_rsv;
4310 4311

	if (!block_rsv)
4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371
int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
			     struct btrfs_block_rsv *dest, u64 num_bytes,
			     int min_factor)
{
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
	u64 min_bytes;

	if (global_rsv->space_info != dest->space_info)
		return -ENOSPC;

	spin_lock(&global_rsv->lock);
	min_bytes = div_factor(global_rsv->size, min_factor);
	if (global_rsv->reserved < min_bytes + num_bytes) {
		spin_unlock(&global_rsv->lock);
		return -ENOSPC;
	}
	global_rsv->reserved -= num_bytes;
	if (global_rsv->reserved < global_rsv->size)
		global_rsv->full = 0;
	spin_unlock(&global_rsv->lock);

	block_rsv_add_bytes(dest, num_bytes, 1);
	return 0;
}

Josef Bacik's avatar
Josef Bacik committed
4372 4373
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
4374
				    struct btrfs_block_rsv *dest, u64 num_bytes)
4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4407
			spin_lock(&space_info->lock);
4408
			space_info->bytes_may_use -= num_bytes;
Josef Bacik's avatar
Josef Bacik committed
4409
			trace_btrfs_space_reservation(fs_info, "space_info",
4410
					space_info->flags, num_bytes, 0);
4411
			space_info->reservation_progress++;
4412
			spin_unlock(&space_info->lock);
4413
		}
Josef Bacik's avatar
Josef Bacik committed
4414
	}
4415
}
4416

4417 4418 4419 4420
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
Josef Bacik's avatar
Josef Bacik committed
4421

4422 4423 4424
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
Josef Bacik's avatar
Josef Bacik committed
4425

4426
	block_rsv_add_bytes(dst, num_bytes, 1);
Josef Bacik's avatar
Josef Bacik committed
4427 4428 4429
	return 0;
}

4430
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
Josef Bacik's avatar
Josef Bacik committed
4431
{
4432 4433
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
4434
	rsv->type = type;
4435 4436
}

4437 4438
struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
					      unsigned short type)
4439 4440 4441
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
Josef Bacik's avatar
Josef Bacik committed
4442

4443 4444 4445
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
Josef Bacik's avatar
Josef Bacik committed
4446

4447
	btrfs_init_block_rsv(block_rsv, type);
4448 4449 4450 4451
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
Josef Bacik's avatar
Josef Bacik committed
4452

4453 4454 4455
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
Josef Bacik's avatar
Josef Bacik committed
4456 4457
	if (!rsv)
		return;
4458 4459
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
Josef Bacik's avatar
Josef Bacik committed
4460 4461
}

4462 4463 4464
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, u64 num_bytes,
			enum btrfs_reserve_flush_enum flush)
Josef Bacik's avatar
Josef Bacik committed
4465
{
4466
	int ret;
Josef Bacik's avatar
Josef Bacik committed
4467

4468 4469
	if (num_bytes == 0)
		return 0;
4470

4471
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4472 4473 4474 4475
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
Josef Bacik's avatar
Josef Bacik committed
4476

4477 4478
	return ret;
}
Josef Bacik's avatar
Josef Bacik committed
4479

4480
int btrfs_block_rsv_check(struct btrfs_root *root,
4481
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4482 4483 4484
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
Josef Bacik's avatar
Josef Bacik committed
4485

4486 4487
	if (!block_rsv)
		return 0;
Josef Bacik's avatar
Josef Bacik committed
4488

4489
	spin_lock(&block_rsv->lock);
4490 4491 4492 4493
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
Josef Bacik's avatar
Josef Bacik committed
4494

4495 4496 4497
	return ret;
}

4498 4499 4500
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv, u64 min_reserved,
			   enum btrfs_reserve_flush_enum flush)
4501 4502 4503 4504 4505 4506 4507 4508 4509
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4510
	if (block_rsv->reserved >= num_bytes)
4511
		ret = 0;
4512
	else
4513 4514
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4515

4516 4517 4518
	if (!ret)
		return 0;

4519
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4520 4521
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4522
		return 0;
4523
	}
Josef Bacik's avatar
Josef Bacik committed
4524

4525
	return ret;
4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
Josef Bacik's avatar
Josef Bacik committed
4543 4544
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
4545 4546 4547
}

/*
4548 4549 4550
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
4551
 */
4552
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
4553
{
4554 4555 4556 4557
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4558
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
4559

4560 4561 4562 4563
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4564

4565 4566
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4567 4568
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4569 4570
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4571

4572 4573 4574
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
4575

4576
	if (num_bytes * 3 > meta_used)
4577
		num_bytes = div64_u64(meta_used, 3);
4578

4579 4580
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
4581

4582 4583 4584 4585 4586
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
4587

4588
	num_bytes = calc_global_metadata_size(fs_info);
4589

4590
	spin_lock(&sinfo->lock);
4591
	spin_lock(&block_rsv->lock);
4592

4593
	block_rsv->size = min_t(u64, num_bytes, 512 * 1024 * 1024);
4594

4595
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4596 4597
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4598 4599 4600 4601

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4602
		sinfo->bytes_may_use += num_bytes;
Josef Bacik's avatar
Josef Bacik committed
4603
		trace_btrfs_space_reservation(fs_info, "space_info",
4604
				      sinfo->flags, num_bytes, 1);
4605 4606
	}

4607 4608
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4609
		sinfo->bytes_may_use -= num_bytes;
Josef Bacik's avatar
Josef Bacik committed
4610
		trace_btrfs_space_reservation(fs_info, "space_info",
4611
				      sinfo->flags, num_bytes, 0);
4612
		sinfo->reservation_progress++;
4613 4614 4615
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4616

4617
	spin_unlock(&block_rsv->lock);
4618
	spin_unlock(&sinfo->lock);
4619 4620
}

4621
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
4622
{
4623
	struct btrfs_space_info *space_info;
4624

4625 4626
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
4627

4628
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4629 4630
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4631 4632
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4633
	fs_info->delayed_block_rsv.space_info = space_info;
4634

4635 4636 4637 4638
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4639 4640
	if (fs_info->quota_root)
		fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
4641
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4642 4643

	update_global_block_rsv(fs_info);
4644 4645
}

4646
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
4647
{
Josef Bacik's avatar
Josef Bacik committed
4648 4649
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4650 4651 4652 4653 4654 4655
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4656 4657
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4658 4659
}

4660 4661
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
4662
{
4663 4664 4665
	if (!trans->block_rsv)
		return;

4666 4667
	if (!trans->bytes_reserved)
		return;
4668

4669
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4670
				      trans->transid, trans->bytes_reserved, 0);
4671
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4672 4673
	trans->bytes_reserved = 0;
}
4674

4675
/* Can only return 0 or -ENOSPC */
4676 4677 4678 4679 4680 4681 4682 4683
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4684 4685 4686
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4687
	 */
Chris Mason's avatar
Chris Mason committed
4688
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
Josef Bacik's avatar
Josef Bacik committed
4689 4690
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4691
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4692 4693
}

4694
void btrfs_orphan_release_metadata(struct inode *inode)
4695
{
4696
	struct btrfs_root *root = BTRFS_I(inode)->root;
Chris Mason's avatar
Chris Mason committed
4697
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
Josef Bacik's avatar
Josef Bacik committed
4698 4699
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4700 4701
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4702

4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720
/*
 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation
 * root: the root of the parent directory
 * rsv: block reservation
 * items: the number of items that we need do reservation
 * qgroup_reserved: used to return the reserved size in qgroup
 *
 * This function is used to reserve the space for snapshot/subvolume
 * creation and deletion. Those operations are different with the
 * common file/directory operations, they change two fs/file trees
 * and root tree, the number of items that the qgroup reserves is
 * different with the free space reservation. So we can not use
 * the space reseravtion mechanism in start_transaction().
 */
int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
				     struct btrfs_block_rsv *rsv,
				     int items,
				     u64 *qgroup_reserved)
4721
{
4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756
	u64 num_bytes;
	int ret;

	if (root->fs_info->quota_enabled) {
		/* One for parent inode, two for dir entries */
		num_bytes = 3 * root->leafsize;
		ret = btrfs_qgroup_reserve(root, num_bytes);
		if (ret)
			return ret;
	} else {
		num_bytes = 0;
	}

	*qgroup_reserved = num_bytes;

	num_bytes = btrfs_calc_trans_metadata_size(root, items);
	rsv->space_info = __find_space_info(root->fs_info,
					    BTRFS_BLOCK_GROUP_METADATA);
	ret = btrfs_block_rsv_add(root, rsv, num_bytes,
				  BTRFS_RESERVE_FLUSH_ALL);
	if (ret) {
		if (*qgroup_reserved)
			btrfs_qgroup_free(root, *qgroup_reserved);
	}

	return ret;
}

void btrfs_subvolume_release_metadata(struct btrfs_root *root,
				      struct btrfs_block_rsv *rsv,
				      u64 qgroup_reserved)
{
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	if (qgroup_reserved)
		btrfs_qgroup_free(root, qgroup_reserved);
4757 4758
}

4759 4760 4761 4762 4763 4764 4765 4766 4767
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4768 4769
static unsigned drop_outstanding_extent(struct inode *inode)
{
4770
	unsigned drop_inode_space = 0;
4771 4772 4773 4774 4775
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

4776
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
4777 4778
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
4779 4780
		drop_inode_space = 1;

4781 4782 4783 4784 4785 4786
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4787
		return drop_inode_space;
4788 4789 4790 4791

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
4792
	return dropped_extents + drop_inode_space;
4793 4794
}

4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
4815
{
4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4851
}
4852

4853 4854 4855 4856
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4857
	u64 to_reserve = 0;
4858
	u64 csum_bytes;
4859
	unsigned nr_extents = 0;
4860
	int extra_reserve = 0;
4861
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
4862
	int ret = 0;
4863
	bool delalloc_lock = true;
4864 4865
	u64 to_free = 0;
	unsigned dropped;
4866

4867 4868 4869 4870 4871 4872
	/* If we are a free space inode we need to not flush since we will be in
	 * the middle of a transaction commit.  We also don't need the delalloc
	 * mutex since we won't race with anybody.  We need this mostly to make
	 * lockdep shut its filthy mouth.
	 */
	if (btrfs_is_free_space_inode(inode)) {
4873
		flush = BTRFS_RESERVE_NO_FLUSH;
4874 4875
		delalloc_lock = false;
	}
4876

4877 4878
	if (flush != BTRFS_RESERVE_NO_FLUSH &&
	    btrfs_transaction_in_commit(root->fs_info))
4879
		schedule_timeout(1);
4880

4881 4882 4883
	if (delalloc_lock)
		mutex_lock(&BTRFS_I(inode)->delalloc_mutex);

4884
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4885

4886 4887 4888 4889
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
4890
	    BTRFS_I(inode)->reserved_extents)
4891 4892
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
4893

4894 4895 4896 4897
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
4898 4899
	if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
		      &BTRFS_I(inode)->runtime_flags)) {
4900
		nr_extents++;
4901
		extra_reserve = 1;
4902
	}
4903 4904

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4905
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4906
	csum_bytes = BTRFS_I(inode)->csum_bytes;
4907
	spin_unlock(&BTRFS_I(inode)->lock);
4908

4909
	if (root->fs_info->quota_enabled) {
4910 4911
		ret = btrfs_qgroup_reserve(root, num_bytes +
					   nr_extents * root->leafsize);
4912 4913 4914
		if (ret)
			goto out_fail;
	}
4915

4916 4917 4918
	ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
	if (unlikely(ret)) {
		if (root->fs_info->quota_enabled)
4919 4920
			btrfs_qgroup_free(root, num_bytes +
						nr_extents * root->leafsize);
4921
		goto out_fail;
4922
	}
4923

4924 4925
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
4926 4927
		set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			&BTRFS_I(inode)->runtime_flags);
4928 4929 4930 4931
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
4932 4933 4934

	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4935

Josef Bacik's avatar
Josef Bacik committed
4936 4937 4938
	if (to_reserve)
		trace_btrfs_space_reservation(root->fs_info,"delalloc",
					      btrfs_ino(inode), to_reserve, 1);
4939 4940 4941
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
4942 4943 4944 4945 4946 4947 4948 4949 4950

out_fail:
	spin_lock(&BTRFS_I(inode)->lock);
	dropped = drop_outstanding_extent(inode);
	/*
	 * If the inodes csum_bytes is the same as the original
	 * csum_bytes then we know we haven't raced with any free()ers
	 * so we can just reduce our inodes csum bytes and carry on.
	 */
4951
	if (BTRFS_I(inode)->csum_bytes == csum_bytes) {
4952
		calc_csum_metadata_size(inode, num_bytes, 0);
4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992
	} else {
		u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes;
		u64 bytes;

		/*
		 * This is tricky, but first we need to figure out how much we
		 * free'd from any free-ers that occured during this
		 * reservation, so we reset ->csum_bytes to the csum_bytes
		 * before we dropped our lock, and then call the free for the
		 * number of bytes that were freed while we were trying our
		 * reservation.
		 */
		bytes = csum_bytes - BTRFS_I(inode)->csum_bytes;
		BTRFS_I(inode)->csum_bytes = csum_bytes;
		to_free = calc_csum_metadata_size(inode, bytes, 0);


		/*
		 * Now we need to see how much we would have freed had we not
		 * been making this reservation and our ->csum_bytes were not
		 * artificially inflated.
		 */
		BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes;
		bytes = csum_bytes - orig_csum_bytes;
		bytes = calc_csum_metadata_size(inode, bytes, 0);

		/*
		 * Now reset ->csum_bytes to what it should be.  If bytes is
		 * more than to_free then we would have free'd more space had we
		 * not had an artificially high ->csum_bytes, so we need to free
		 * the remainder.  If bytes is the same or less then we don't
		 * need to do anything, the other free-ers did the correct
		 * thing.
		 */
		BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes;
		if (bytes > to_free)
			to_free = bytes - to_free;
		else
			to_free = 0;
	}
4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
	spin_unlock(&BTRFS_I(inode)->lock);
	if (dropped)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);

	if (to_free) {
		btrfs_block_rsv_release(root, block_rsv, to_free);
		trace_btrfs_space_reservation(root->fs_info, "delalloc",
					      btrfs_ino(inode), to_free, 0);
	}
	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
	return ret;
5005 5006
}

5007 5008 5009 5010 5011 5012 5013 5014 5015
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
5016 5017 5018
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
5019 5020
	u64 to_free = 0;
	unsigned dropped;
5021 5022

	num_bytes = ALIGN(num_bytes, root->sectorsize);
5023
	spin_lock(&BTRFS_I(inode)->lock);
5024
	dropped = drop_outstanding_extent(inode);
5025

5026 5027
	if (num_bytes)
		to_free = calc_csum_metadata_size(inode, num_bytes, 0);
5028
	spin_unlock(&BTRFS_I(inode)->lock);
5029 5030
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
5031

Josef Bacik's avatar
Josef Bacik committed
5032 5033
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
5034 5035 5036 5037 5038
	if (root->fs_info->quota_enabled) {
		btrfs_qgroup_free(root, num_bytes +
					dropped * root->leafsize);
	}

5039 5040 5041 5042
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
5058 5059 5060 5061 5062
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
5063
	if (ret)
5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
5088 5089 5090 5091
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
5092 5093
}

5094
static int update_block_group(struct btrfs_root *root,
5095
			      u64 bytenr, u64 num_bytes, int alloc)
5096
{
5097
	struct btrfs_block_group_cache *cache = NULL;
5098
	struct btrfs_fs_info *info = root->fs_info;
5099
	u64 total = num_bytes;
5100
	u64 old_val;
5101
	u64 byte_in_group;
5102
	int factor;
Chris Mason's avatar
Chris Mason committed
5103

5104
	/* block accounting for super block */
5105
	spin_lock(&info->delalloc_root_lock);
5106
	old_val = btrfs_super_bytes_used(info->super_copy);
5107 5108 5109 5110
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
5111
	btrfs_set_super_bytes_used(info->super_copy, old_val);
5112
	spin_unlock(&info->delalloc_root_lock);
5113

5114
	while (total) {
5115
		cache = btrfs_lookup_block_group(info, bytenr);
5116
		if (!cache)
5117
			return -ENOENT;
5118 5119 5120 5121 5122 5123
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
5124 5125 5126 5127 5128 5129 5130
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
5131
			cache_block_group(cache, 1);
5132

5133 5134
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
5135

5136
		spin_lock(&cache->space_info->lock);
5137
		spin_lock(&cache->lock);
5138

5139
		if (btrfs_test_opt(root, SPACE_CACHE) &&
5140 5141 5142
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

5143
		cache->dirty = 1;
5144
		old_val = btrfs_block_group_used(&cache->item);
5145
		num_bytes = min(total, cache->key.offset - byte_in_group);
5146
		if (alloc) {
5147
			old_val += num_bytes;
5148 5149 5150
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
5151 5152
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
5153
			spin_unlock(&cache->lock);
5154
			spin_unlock(&cache->space_info->lock);
5155
		} else {
5156
			old_val -= num_bytes;
5157
			btrfs_set_block_group_used(&cache->item, old_val);
5158 5159
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
5160
			cache->space_info->bytes_used -= num_bytes;
5161
			cache->space_info->disk_used -= num_bytes * factor;
5162
			spin_unlock(&cache->lock);
5163
			spin_unlock(&cache->space_info->lock);
5164

5165 5166 5167
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
5168
		}
5169
		btrfs_put_block_group(cache);
5170 5171
		total -= num_bytes;
		bytenr += num_bytes;
5172 5173 5174
	}
	return 0;
}
5175

5176 5177
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
5178
	struct btrfs_block_group_cache *cache;
5179
	u64 bytenr;
5180

5181 5182 5183 5184 5185 5186 5187
	spin_lock(&root->fs_info->block_group_cache_lock);
	bytenr = root->fs_info->first_logical_byte;
	spin_unlock(&root->fs_info->block_group_cache_lock);

	if (bytenr < (u64)-1)
		return bytenr;

5188 5189
	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
5190
		return 0;
5191

5192
	bytenr = cache->key.objectid;
5193
	btrfs_put_block_group(cache);
5194 5195

	return bytenr;
5196 5197
}

5198 5199 5200
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
5201
{
5202 5203 5204 5205 5206 5207 5208 5209 5210 5211
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
5212

5213 5214 5215 5216
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
5217

5218 5219 5220 5221 5222 5223 5224
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
5225

5226
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5227
	BUG_ON(!cache); /* Logic error */
5228 5229 5230 5231

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
5232 5233 5234
	return 0;
}

5235
/*
5236 5237
 * this function must be called within transaction
 */
5238
int btrfs_pin_extent_for_log_replay(struct btrfs_root *root,
5239 5240 5241
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;
5242
	int ret;
5243 5244

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5245 5246
	if (!cache)
		return -EINVAL;
5247 5248 5249 5250 5251 5252 5253

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
5254
	cache_block_group(cache, 1);
5255 5256 5257 5258

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
5259
	ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
5260
	btrfs_put_block_group(cache);
5261
	return ret;
5262 5263
}

5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337
static int __exclude_logged_extent(struct btrfs_root *root, u64 start, u64 num_bytes)
{
	int ret;
	struct btrfs_block_group_cache *block_group;
	struct btrfs_caching_control *caching_ctl;

	block_group = btrfs_lookup_block_group(root->fs_info, start);
	if (!block_group)
		return -EINVAL;

	cache_block_group(block_group, 0);
	caching_ctl = get_caching_control(block_group);

	if (!caching_ctl) {
		/* Logic error */
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			if (ret)
				goto out_lock;

			num_bytes = (start + num_bytes) -
				caching_ctl->progress;
			start = caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
		}
out_lock:
		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}
	btrfs_put_block_group(block_group);
	return ret;
}

int btrfs_exclude_logged_extents(struct btrfs_root *log,
				 struct extent_buffer *eb)
{
	struct btrfs_file_extent_item *item;
	struct btrfs_key key;
	int found_type;
	int i;

	if (!btrfs_fs_incompat(log->fs_info, MIXED_GROUPS))
		return 0;

	for (i = 0; i < btrfs_header_nritems(eb); i++) {
		btrfs_item_key_to_cpu(eb, &key, i);
		if (key.type != BTRFS_EXTENT_DATA_KEY)
			continue;
		item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
		found_type = btrfs_file_extent_type(eb, item);
		if (found_type == BTRFS_FILE_EXTENT_INLINE)
			continue;
		if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
			continue;
		key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
		key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
		__exclude_logged_extent(log, key.objectid, key.offset);
	}

	return 0;
}

5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
5359
 */
5360 5361
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
5362
{
5363
	struct btrfs_space_info *space_info = cache->space_info;
5364
	int ret = 0;
5365

5366 5367 5368
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
5369 5370 5371
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
5372 5373 5374
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
Josef Bacik's avatar
Josef Bacik committed
5375
				trace_btrfs_space_reservation(cache->fs_info,
5376 5377
						"space_info", space_info->flags,
						num_bytes, 0);
5378 5379
				space_info->bytes_may_use -= num_bytes;
			}
5380
		}
5381 5382 5383 5384 5385 5386
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
5387
	}
5388 5389
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
5390
	return ret;
5391
}
5392

5393
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
5394
				struct btrfs_root *root)
5395 5396
{
	struct btrfs_fs_info *fs_info = root->fs_info;
5397 5398 5399
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
5400

5401
	down_write(&fs_info->extent_commit_sem);
5402

5403 5404 5405 5406 5407 5408 5409
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
5410
		} else {
5411
			cache->last_byte_to_unpin = caching_ctl->progress;
5412 5413
		}
	}
5414 5415 5416 5417 5418 5419 5420

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
5421 5422

	update_global_block_rsv(fs_info);
5423 5424
}

5425
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
Chris Mason's avatar
Chris Mason committed
5426
{
5427 5428
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
5429 5430
	struct btrfs_space_info *space_info;
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5431
	u64 len;
5432
	bool readonly;
Chris Mason's avatar
Chris Mason committed
5433

5434
	while (start <= end) {
5435
		readonly = false;
5436 5437 5438 5439 5440
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
5441
			BUG_ON(!cache); /* Logic error */
5442 5443 5444 5445 5446 5447 5448 5449 5450 5451
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

5452
		start += len;
5453
		space_info = cache->space_info;
5454

5455
		spin_lock(&space_info->lock);
5456 5457
		spin_lock(&cache->lock);
		cache->pinned -= len;
5458 5459 5460 5461 5462
		space_info->bytes_pinned -= len;
		if (cache->ro) {
			space_info->bytes_readonly += len;
			readonly = true;
		}
5463
		spin_unlock(&cache->lock);
5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476
		if (!readonly && global_rsv->space_info == space_info) {
			spin_lock(&global_rsv->lock);
			if (!global_rsv->full) {
				len = min(len, global_rsv->size -
					  global_rsv->reserved);
				global_rsv->reserved += len;
				space_info->bytes_may_use += len;
				if (global_rsv->reserved >= global_rsv->size)
					global_rsv->full = 1;
			}
			spin_unlock(&global_rsv->lock);
		}
		spin_unlock(&space_info->lock);
Chris Mason's avatar
Chris Mason committed
5477
	}
5478 5479 5480

	if (cache)
		btrfs_put_block_group(cache);
Chris Mason's avatar
Chris Mason committed
5481 5482 5483 5484
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5485
			       struct btrfs_root *root)
5486
{
5487 5488
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
5489 5490
	u64 start;
	u64 end;
5491 5492
	int ret;

5493 5494 5495
	if (trans->aborted)
		return 0;

5496 5497 5498 5499 5500
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

5501
	while (1) {
5502
		ret = find_first_extent_bit(unpin, 0, &start, &end,
5503
					    EXTENT_DIRTY, NULL);
5504
		if (ret)
5505
			break;
5506

5507 5508 5509
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
5510

5511
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
5512
		unpin_extent_range(root, start, end);
5513
		cond_resched();
5514
	}
5515

Chris Mason's avatar
Chris Mason committed
5516 5517 5518
	return 0;
}

5519 5520 5521 5522 5523 5524
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
5525
{
Chris Mason's avatar
Chris Mason committed
5526
	struct btrfs_key key;
5527
	struct btrfs_path *path;
5528 5529
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
5530
	struct extent_buffer *leaf;
5531 5532
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
5533
	int ret;
5534
	int is_data;
5535 5536 5537
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
5538 5539
	u32 item_size;
	u64 refs;
5540 5541
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
5542

5543
	path = btrfs_alloc_path();
5544 5545
	if (!path)
		return -ENOMEM;
5546

5547
	path->reada = 1;
5548
	path->leave_spinning = 1;
5549 5550 5551 5552

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

5553 5554 5555
	if (is_data)
		skinny_metadata = 0;

5556 5557 5558 5559
	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
5560
	if (ret == 0) {
5561
		extent_slot = path->slots[0];
5562 5563
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
5564
					      extent_slot);
5565
			if (key.objectid != bytenr)
5566
				break;
5567 5568
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
5569 5570 5571
				found_extent = 1;
				break;
			}
5572 5573 5574 5575 5576
			if (key.type == BTRFS_METADATA_ITEM_KEY &&
			    key.offset == owner_objectid) {
				found_extent = 1;
				break;
			}
5577 5578
			if (path->slots[0] - extent_slot > 5)
				break;
5579
			extent_slot--;
5580
		}
5581 5582 5583 5584 5585
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
5586
		if (!found_extent) {
5587
			BUG_ON(iref);
5588
			ret = remove_extent_backref(trans, extent_root, path,
5589 5590
						    NULL, refs_to_drop,
						    is_data);
5591 5592 5593 5594
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5595
			btrfs_release_path(path);
5596
			path->leave_spinning = 1;
5597 5598 5599 5600 5601

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

5602 5603 5604 5605 5606
			if (!is_data && skinny_metadata) {
				key.type = BTRFS_METADATA_ITEM_KEY;
				key.offset = owner_objectid;
			}

5607 5608
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631
			if (ret > 0 && skinny_metadata && path->slots[0]) {
				/*
				 * Couldn't find our skinny metadata item,
				 * see if we have ye olde extent item.
				 */
				path->slots[0]--;
				btrfs_item_key_to_cpu(path->nodes[0], &key,
						      path->slots[0]);
				if (key.objectid == bytenr &&
				    key.type == BTRFS_EXTENT_ITEM_KEY &&
				    key.offset == num_bytes)
					ret = 0;
			}

			if (ret > 0 && skinny_metadata) {
				skinny_metadata = false;
				key.type = BTRFS_EXTENT_ITEM_KEY;
				key.offset = num_bytes;
				btrfs_release_path(path);
				ret = btrfs_search_slot(trans, extent_root,
							&key, path, -1, 1);
			}

5632
			if (ret) {
5633 5634
				btrfs_err(info, "umm, got %d back from search, was looking for %llu",
					ret, (unsigned long long)bytenr);
5635 5636 5637
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5638
			}
5639 5640 5641 5642
			if (ret < 0) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5643 5644
			extent_slot = path->slots[0];
		}
5645
	} else if (ret == -ENOENT) {
5646 5647
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
5648 5649 5650 5651 5652 5653 5654
		btrfs_err(info,
			"unable to find ref byte nr %llu parent %llu root %llu  owner %llu offset %llu",
			(unsigned long long)bytenr,
			(unsigned long long)parent,
			(unsigned long long)root_objectid,
			(unsigned long long)owner_objectid,
			(unsigned long long)owner_offset);
5655
	} else {
5656 5657
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
5658
	}
5659 5660

	leaf = path->nodes[0];
5661 5662 5663 5664 5665 5666
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5667 5668 5669 5670
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5671

5672
		btrfs_release_path(path);
5673 5674 5675 5676 5677 5678 5679 5680 5681
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
5682 5683
			btrfs_err(info, "umm, got %d back from search, was looking for %llu",
				ret, (unsigned long long)bytenr);
5684 5685
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5686 5687 5688 5689 5690
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

5691 5692 5693 5694 5695 5696
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5697
	ei = btrfs_item_ptr(leaf, extent_slot,
5698
			    struct btrfs_extent_item);
5699 5700
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
	    key.type == BTRFS_EXTENT_ITEM_KEY) {
5701 5702 5703 5704 5705
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5706

5707
	refs = btrfs_extent_refs(leaf, ei);
5708 5709 5710 5711 5712 5713 5714
	if (refs < refs_to_drop) {
		btrfs_err(info, "trying to drop %d refs but we only have %Lu "
			  "for bytenr %Lu\n", refs_to_drop, refs, bytenr);
		ret = -EINVAL;
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
	}
5715
	refs -= refs_to_drop;
5716

5717 5718 5719 5720 5721 5722
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5723
		 */
5724 5725 5726 5727 5728 5729 5730 5731 5732 5733
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
5734 5735 5736 5737
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5738
		}
5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
5750
		}
5751

5752 5753
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
5754 5755 5756 5757
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5758
		btrfs_release_path(path);
5759

5760
		if (is_data) {
5761
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5762 5763 5764 5765
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5766 5767
		}

5768
		ret = update_block_group(root, bytenr, num_bytes, 0);
5769 5770 5771 5772
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5773
	}
5774
out:
5775
	btrfs_free_path(path);
5776 5777 5778
	return ret;
}

5779
/*
5780
 * when we free an block, it is possible (and likely) that we free the last
5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
5792
	int ret = 0;
5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

5810 5811 5812
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
5813
		btrfs_free_delayed_extent_op(head->extent_op);
5814 5815 5816
		head->extent_op = NULL;
	}

5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
5830

5831 5832 5833 5834 5835 5836
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
5837 5838 5839 5840 5841
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
5842 5843
	spin_unlock(&delayed_refs->lock);

5844 5845 5846 5847 5848
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
5849
	btrfs_put_delayed_ref(&head->node);
5850
	return ret;
5851 5852 5853 5854 5855
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

5856 5857 5858
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
5859
			   u64 parent, int last_ref)
5860 5861 5862 5863 5864
{
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5865 5866 5867 5868
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
5869
					BTRFS_DROP_DELAYED_REF, NULL, 0);
5870
		BUG_ON(ret); /* -ENOMEM */
5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881
	}

	if (!last_ref)
		return;

	cache = btrfs_lookup_block_group(root->fs_info, buf->start);

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
5882
				goto out;
5883 5884 5885 5886
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
5887
			goto out;
5888 5889 5890 5891 5892
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
5893
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5894 5895
	}
out:
5896 5897 5898 5899 5900
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5901 5902 5903
	btrfs_put_block_group(cache);
}

5904
/* Can return -ENOMEM */
5905 5906 5907
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
		      u64 owner, u64 offset, int for_cow)
5908 5909
{
	int ret;
5910
	struct btrfs_fs_info *fs_info = root->fs_info;
5911

5912 5913 5914 5915
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
5916 5917
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5918
		/* unlocks the pinned mutex */
5919
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
5920
		ret = 0;
5921
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
5922 5923
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
5924
					parent, root_objectid, (int)owner,
5925
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5926
	} else {
5927 5928 5929 5930 5931
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
						NULL, for_cow);
5932
	}
5933 5934 5935
	return ret;
}

David Woodhouse's avatar
David Woodhouse committed
5936 5937 5938
static u64 stripe_align(struct btrfs_root *root,
			struct btrfs_block_group_cache *cache,
			u64 val, u64 num_bytes)
5939
{
5940
	u64 ret = ALIGN(val, root->stripesize);
5941 5942 5943
	return ret;
}

5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
5959
	struct btrfs_caching_control *caching_ctl;
5960

5961 5962
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
5963 5964
		return 0;

5965
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5966
		   (cache->free_space_ctl->free_space >= num_bytes));
5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
5984 5985 5986
	return 0;
}

5987
int __get_raid_index(u64 flags)
5988
{
5989
	if (flags & BTRFS_BLOCK_GROUP_RAID10)
5990
		return BTRFS_RAID_RAID10;
5991
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5992
		return BTRFS_RAID_RAID1;
5993
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
5994
		return BTRFS_RAID_DUP;
5995
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5996
		return BTRFS_RAID_RAID0;
David Woodhouse's avatar
David Woodhouse committed
5997
	else if (flags & BTRFS_BLOCK_GROUP_RAID5)
5998
		return BTRFS_RAID_RAID5;
David Woodhouse's avatar
David Woodhouse committed
5999
	else if (flags & BTRFS_BLOCK_GROUP_RAID6)
6000
		return BTRFS_RAID_RAID6;
6001

6002
	return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */
6003 6004
}

6005 6006
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
6007
	return __get_raid_index(cache->flags);
6008 6009
}

6010
enum btrfs_loop_type {
6011 6012 6013 6014
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
6015 6016
};

6017 6018 6019 6020
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
6021
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
6022 6023 6024
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
6025
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
6026 6027 6028
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
6029
				     u64 flags)
6030
{
6031
	int ret = 0;
6032
	struct btrfs_root *root = orig_root->fs_info->extent_root;
6033
	struct btrfs_free_cluster *last_ptr = NULL;
6034
	struct btrfs_block_group_cache *block_group = NULL;
6035
	struct btrfs_block_group_cache *used_block_group;
6036
	u64 search_start = 0;
6037
	int empty_cluster = 2 * 1024 * 1024;
6038
	struct btrfs_space_info *space_info;
6039
	int loop = 0;
6040 6041
	int index = __get_raid_index(flags);
	int alloc_type = (flags & BTRFS_BLOCK_GROUP_DATA) ?
6042
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
6043
	bool found_uncached_bg = false;
6044
	bool failed_cluster_refill = false;
6045
	bool failed_alloc = false;
6046
	bool use_cluster = true;
6047
	bool have_caching_bg = false;
6048

6049
	WARN_ON(num_bytes < root->sectorsize);
6050
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
6051 6052
	ins->objectid = 0;
	ins->offset = 0;
6053

6054
	trace_find_free_extent(orig_root, num_bytes, empty_size, flags);
6055

6056
	space_info = __find_space_info(root->fs_info, flags);
6057
	if (!space_info) {
6058
		btrfs_err(root->fs_info, "No space info for %llu", flags);
6059 6060
		return -ENOSPC;
	}
6061

6062 6063 6064 6065 6066 6067 6068
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

6069
	if (flags & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
6070
		last_ptr = &root->fs_info->meta_alloc_cluster;
6071 6072
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
6073 6074
	}

6075
	if ((flags & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
6076
	    btrfs_test_opt(root, SSD)) {
6077 6078
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
6079

6080
	if (last_ptr) {
6081 6082 6083 6084
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
6085
	}
6086

6087
	search_start = max(search_start, first_logical_byte(root, 0));
6088
	search_start = max(search_start, hint_byte);
6089

6090
	if (!last_ptr)
6091 6092
		empty_cluster = 0;

6093 6094 6095
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
6096
		used_block_group = block_group;
6097 6098 6099
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
6100 6101 6102
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
6103
		 */
6104
		if (block_group && block_group_bits(block_group, flags) &&
6105
		    block_group->cached != BTRFS_CACHE_NO) {
6106
			down_read(&space_info->groups_sem);
6107 6108 6109 6110 6111 6112 6113 6114 6115 6116
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
6117
			} else {
6118
				index = get_block_group_index(block_group);
6119
				goto have_block_group;
6120
			}
6121
		} else if (block_group) {
6122
			btrfs_put_block_group(block_group);
6123
		}
6124
	}
6125
search:
6126
	have_caching_bg = false;
6127
	down_read(&space_info->groups_sem);
6128 6129
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
6130
		u64 offset;
6131
		int cached;
6132

6133
		used_block_group = block_group;
6134
		btrfs_get_block_group(block_group);
6135
		search_start = block_group->key.objectid;
6136

6137 6138 6139 6140 6141
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
6142
		if (!block_group_bits(block_group, flags)) {
6143 6144
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
David Woodhouse's avatar
David Woodhouse committed
6145 6146
				BTRFS_BLOCK_GROUP_RAID5 |
				BTRFS_BLOCK_GROUP_RAID6 |
6147 6148 6149 6150 6151 6152 6153
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
6154
			if ((flags & extra) && !(block_group->flags & extra))
6155 6156 6157
				goto loop;
		}

6158
have_block_group:
6159 6160 6161
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
			found_uncached_bg = true;
6162
			ret = cache_block_group(block_group, 0);
6163 6164
			BUG_ON(ret < 0);
			ret = 0;
6165 6166
		}

6167
		if (unlikely(block_group->ro))
6168
			goto loop;
6169

6170
		/*
6171 6172
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
6173
		 */
6174
		if (last_ptr) {
6175
			unsigned long aligned_cluster;
6176 6177 6178 6179 6180
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
6181 6182 6183 6184
			used_block_group = last_ptr->block_group;
			if (used_block_group != block_group &&
			    (!used_block_group ||
			     used_block_group->ro ||
6185
			     !block_group_bits(used_block_group, flags))) {
6186
				used_block_group = block_group;
6187
				goto refill_cluster;
6188 6189 6190 6191
			}

			if (used_block_group != block_group)
				btrfs_get_block_group(used_block_group);
6192

6193 6194
			offset = btrfs_alloc_from_cluster(used_block_group,
			  last_ptr, num_bytes, used_block_group->key.objectid);
6195 6196 6197
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
6198 6199
				trace_btrfs_reserve_extent_cluster(root,
					block_group, search_start, num_bytes);
6200 6201 6202
				goto checks;
			}

6203 6204 6205 6206
			WARN_ON(last_ptr->block_group != used_block_group);
			if (used_block_group != block_group) {
				btrfs_put_block_group(used_block_group);
				used_block_group = block_group;
6207
			}
6208
refill_cluster:
6209
			BUG_ON(used_block_group != block_group);
6210 6211 6212 6213 6214 6215 6216 6217
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
6218 6219 6220 6221 6222 6223 6224 6225 6226
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
			    last_ptr->block_group != block_group) {
6227 6228 6229 6230
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

6231 6232 6233 6234 6235 6236
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

6237 6238 6239 6240 6241
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

6242 6243 6244 6245
			aligned_cluster = max_t(unsigned long,
						empty_cluster + empty_size,
					      block_group->full_stripe_len);

6246
			/* allocate a cluster in this block group */
6247
			ret = btrfs_find_space_cluster(trans, root,
6248
					       block_group, last_ptr,
6249
					       search_start, num_bytes,
6250
					       aligned_cluster);
6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
6262 6263 6264
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
6265 6266
					goto checks;
				}
6267 6268
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
6269 6270
				spin_unlock(&last_ptr->refill_lock);

6271
				failed_cluster_refill = true;
6272 6273 6274
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
6275
			}
6276

6277 6278 6279 6280 6281 6282
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
6283
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
6284
			spin_unlock(&last_ptr->refill_lock);
6285
			goto loop;
6286 6287
		}

6288
unclustered_alloc:
6289 6290 6291 6292 6293 6294 6295 6296 6297
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

6298 6299
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
6311
			wait_block_group_cache_progress(block_group,
6312 6313
						num_bytes + empty_size);
			failed_alloc = true;
6314
			goto have_block_group;
6315
		} else if (!offset) {
6316 6317
			if (!cached)
				have_caching_bg = true;
6318
			goto loop;
6319
		}
6320
checks:
David Woodhouse's avatar
David Woodhouse committed
6321 6322
		search_start = stripe_align(root, used_block_group,
					    offset, num_bytes);
6323

6324 6325
		/* move on to the next group */
		if (search_start + num_bytes >
6326 6327
		    used_block_group->key.objectid + used_block_group->key.offset) {
			btrfs_add_free_space(used_block_group, offset, num_bytes);
6328
			goto loop;
6329
		}
6330

6331
		if (offset < search_start)
6332
			btrfs_add_free_space(used_block_group, offset,
6333 6334
					     search_start - offset);
		BUG_ON(offset > search_start);
6335

6336
		ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
6337
						  alloc_type);
6338
		if (ret == -EAGAIN) {
6339
			btrfs_add_free_space(used_block_group, offset, num_bytes);
6340
			goto loop;
6341
		}
6342

6343
		/* we are all good, lets return */
6344 6345
		ins->objectid = search_start;
		ins->offset = num_bytes;
6346

6347 6348
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
6349 6350
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
6351
		btrfs_put_block_group(block_group);
6352 6353
		break;
loop:
6354
		failed_cluster_refill = false;
6355
		failed_alloc = false;
6356
		BUG_ON(index != get_block_group_index(block_group));
6357 6358
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
6359
		btrfs_put_block_group(block_group);
6360 6361 6362
	}
	up_read(&space_info->groups_sem);

6363 6364 6365
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

6366 6367 6368
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

6369
	/*
6370 6371
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
6372 6373 6374 6375
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
6376
	 */
6377
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
6378
		index = 0;
6379
		loop++;
6380
		if (loop == LOOP_ALLOC_CHUNK) {
6381
			ret = do_chunk_alloc(trans, root, flags,
6382 6383 6384 6385 6386 6387 6388 6389 6390
					     CHUNK_ALLOC_FORCE);
			/*
			 * Do not bail out on ENOSPC since we
			 * can do more things.
			 */
			if (ret < 0 && ret != -ENOSPC) {
				btrfs_abort_transaction(trans,
							root, ret);
				goto out;
6391
			}
6392 6393
		}

6394 6395 6396
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
6397
		}
6398 6399

		goto search;
6400 6401
	} else if (!ins->objectid) {
		ret = -ENOSPC;
6402
	} else if (ins->objectid) {
6403
		ret = 0;
6404
	}
6405
out:
6406

6407
	return ret;
6408
}
6409

Josef Bacik's avatar
Josef Bacik committed
6410 6411
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
6412 6413
{
	struct btrfs_block_group_cache *cache;
6414
	int index = 0;
6415

Josef Bacik's avatar
Josef Bacik committed
6416
	spin_lock(&info->lock);
6417 6418
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
6419
	       (unsigned long long)(info->total_bytes - info->bytes_used -
Josef Bacik's avatar
Josef Bacik committed
6420
				    info->bytes_pinned - info->bytes_reserved -
6421
				    info->bytes_readonly),
6422
	       (info->full) ? "" : "not ");
6423 6424
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
6425
	       (unsigned long long)info->total_bytes,
6426
	       (unsigned long long)info->bytes_used,
6427
	       (unsigned long long)info->bytes_pinned,
6428
	       (unsigned long long)info->bytes_reserved,
6429
	       (unsigned long long)info->bytes_may_use,
6430
	       (unsigned long long)info->bytes_readonly);
Josef Bacik's avatar
Josef Bacik committed
6431 6432 6433 6434
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
6435

6436
	down_read(&info->groups_sem);
6437 6438
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
6439
		spin_lock(&cache->lock);
6440
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
6441 6442 6443 6444
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
6445 6446
		       (unsigned long long)cache->reserved,
		       cache->ro ? "[readonly]" : "");
6447 6448 6449
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
6450 6451
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
6452
	up_read(&info->groups_sem);
6453
}
6454

6455 6456 6457 6458
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
6459
			 struct btrfs_key *ins, int is_data)
6460
{
6461
	bool final_tried = false;
6462
	u64 flags;
6463
	int ret;
6464

6465
	flags = btrfs_get_alloc_profile(root, is_data);
6466
again:
6467 6468
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
6469
			       hint_byte, ins, flags);
6470

6471 6472 6473
	if (ret == -ENOSPC) {
		if (!final_tried) {
			num_bytes = num_bytes >> 1;
6474
			num_bytes = round_down(num_bytes, root->sectorsize);
6475 6476 6477 6478 6479 6480 6481
			num_bytes = max(num_bytes, min_alloc_size);
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

6482
			sinfo = __find_space_info(root->fs_info, flags);
6483
			btrfs_err(root->fs_info, "allocation failed flags %llu, wanted %llu",
6484
				(unsigned long long)flags,
6485
				(unsigned long long)num_bytes);
6486 6487
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
6488
		}
6489
	}
6490

6491 6492
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

6493
	return ret;
6494 6495
}

6496 6497
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len, int pin)
6498
{
6499
	struct btrfs_block_group_cache *cache;
6500
	int ret = 0;
6501 6502 6503

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
6504 6505
		btrfs_err(root->fs_info, "Unable to find block group for %llu",
			(unsigned long long)start);
6506 6507
		return -ENOSPC;
	}
6508

6509 6510
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
6511

6512 6513 6514 6515 6516 6517
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
	}
6518
	btrfs_put_block_group(cache);
6519

6520 6521
	trace_btrfs_reserved_extent_free(root, start, len);

6522 6523 6524
	return ret;
}

6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536
int btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 0);
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 1);
}

6537 6538 6539 6540 6541
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
6542 6543
{
	int ret;
6544
	struct btrfs_fs_info *fs_info = root->fs_info;
6545
	struct btrfs_extent_item *extent_item;
6546
	struct btrfs_extent_inline_ref *iref;
6547
	struct btrfs_path *path;
6548 6549 6550
	struct extent_buffer *leaf;
	int type;
	u32 size;
6551

6552 6553 6554 6555
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
6556

6557
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6558 6559

	path = btrfs_alloc_path();
Tsutomu Itoh's avatar
Tsutomu Itoh committed
6560 6561
	if (!path)
		return -ENOMEM;
6562

6563
	path->leave_spinning = 1;
6564 6565
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6566 6567 6568 6569
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6570

6571 6572
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
6573
				     struct btrfs_extent_item);
6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
6594 6595

	btrfs_mark_buffer_dirty(path->nodes[0]);
6596
	btrfs_free_path(path);
6597

6598
	ret = update_block_group(root, ins->objectid, ins->offset, 1);
6599
	if (ret) { /* -ENOENT, logic error */
6600 6601 6602
		btrfs_err(fs_info, "update block group failed for %llu %llu",
			(unsigned long long)ins->objectid,
			(unsigned long long)ins->offset);
6603 6604
		BUG();
	}
6605 6606 6607
	return ret;
}

6608 6609 6610 6611 6612
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
6613 6614
{
	int ret;
6615 6616 6617 6618 6619 6620
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
6621 6622 6623 6624 6625 6626
	u32 size = sizeof(*extent_item) + sizeof(*iref);
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);

	if (!skinny_metadata)
		size += sizeof(*block_info);
6627

6628
	path = btrfs_alloc_path();
6629 6630
	if (!path)
		return -ENOMEM;
6631

6632 6633 6634
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6635 6636 6637 6638
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6639 6640 6641 6642 6643 6644 6645 6646 6647

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);

6648 6649 6650 6651 6652 6653 6654 6655
	if (skinny_metadata) {
		iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	} else {
		block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
		btrfs_set_tree_block_key(leaf, block_info, key);
		btrfs_set_tree_block_level(leaf, block_info, level);
		iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	}
6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670

	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

6671
	ret = update_block_group(root, ins->objectid, root->leafsize, 1);
6672
	if (ret) { /* -ENOENT, logic error */
6673 6674 6675
		btrfs_err(fs_info, "update block group failed for %llu %llu",
			(unsigned long long)ins->objectid,
			(unsigned long long)ins->offset);
6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

6690 6691 6692 6693
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6694 6695
	return ret;
}
6696 6697 6698 6699 6700 6701

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
6702 6703 6704 6705
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
6706 6707 6708
{
	int ret;
	struct btrfs_block_group_cache *block_group;
6709

6710 6711 6712 6713 6714 6715
	/*
	 * Mixed block groups will exclude before processing the log so we only
	 * need to do the exlude dance if this fs isn't mixed.
	 */
	if (!btrfs_fs_incompat(root->fs_info, MIXED_GROUPS)) {
		ret = __exclude_logged_extent(root, ins->objectid, ins->offset);
6716
		if (ret)
6717
			return ret;
6718 6719
	}

6720 6721 6722 6723
	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
	if (!block_group)
		return -EINVAL;

6724 6725
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
6726
	BUG_ON(ret); /* logic error */
6727 6728
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
6729
	btrfs_put_block_group(block_group);
6730 6731 6732
	return ret;
}

6733 6734 6735
static struct extent_buffer *
btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u32 blocksize, int level)
6736 6737 6738 6739 6740 6741 6742
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
6743
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6744 6745
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
6746
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6747 6748

	btrfs_set_lock_blocking(buf);
6749
	btrfs_set_buffer_uptodate(buf);
6750

6751
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6752 6753 6754 6755 6756 6757 6758 6759 6760 6761
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
6762 6763
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6764
			 buf->start + buf->len - 1, GFP_NOFS);
6765
	}
6766
	trans->blocks_used++;
6767
	/* this returns a buffer locked for blocking */
6768 6769 6770
	return buf;
}

6771 6772 6773 6774 6775
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
6776
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6777
	int ret;
6778
	bool global_updated = false;
6779 6780 6781

	block_rsv = get_block_rsv(trans, root);

6782 6783
	if (unlikely(block_rsv->size == 0))
		goto try_reserve;
6784
again:
6785 6786 6787 6788
	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;

6789 6790 6791
	if (block_rsv->failfast)
		return ERR_PTR(ret);

6792 6793 6794 6795 6796 6797
	if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
		global_updated = true;
		update_global_block_rsv(root->fs_info);
		goto again;
	}

6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812
	if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL * 10,
				/*DEFAULT_RATELIMIT_BURST*/ 1);
		if (__ratelimit(&_rs))
			WARN(1, KERN_DEBUG
				"btrfs: block rsv returned %d\n", ret);
	}
try_reserve:
	ret = reserve_metadata_bytes(root, block_rsv, blocksize,
				     BTRFS_RESERVE_NO_FLUSH);
	if (!ret)
		return block_rsv;
	/*
	 * If we couldn't reserve metadata bytes try and use some from
6813 6814
	 * the global reserve if its space type is the same as the global
	 * reservation.
6815
	 */
6816 6817
	if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
	    block_rsv->space_info == global_rsv->space_info) {
6818 6819 6820 6821 6822
		ret = block_rsv_use_bytes(global_rsv, blocksize);
		if (!ret)
			return global_rsv;
	}
	return ERR_PTR(ret);
6823 6824
}

Josef Bacik's avatar
Josef Bacik committed
6825 6826
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
6827 6828
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
Josef Bacik's avatar
Josef Bacik committed
6829
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6830 6831
}

6832
/*
6833 6834 6835 6836
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
6837 6838
 * returns the tree buffer or NULL.
 */
6839
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6840 6841 6842
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
6843
					u64 hint, u64 empty_size)
6844
{
Chris Mason's avatar
Chris Mason committed
6845
	struct btrfs_key ins;
6846
	struct btrfs_block_rsv *block_rsv;
6847
	struct extent_buffer *buf;
6848 6849
	u64 flags = 0;
	int ret;
6850 6851
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
6852

6853 6854 6855 6856 6857
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6858
				   empty_size, hint, &ins, 0);
6859
	if (ret) {
Josef Bacik's avatar
Josef Bacik committed
6860
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6861
		return ERR_PTR(ret);
6862
	}
6863

6864 6865
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
6866
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6867 6868 6869 6870 6871 6872 6873 6874 6875 6876

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
6877
		extent_op = btrfs_alloc_delayed_extent_op();
6878
		BUG_ON(!extent_op); /* -ENOMEM */
6879 6880 6881 6882 6883
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
6884 6885 6886 6887
		if (skinny_metadata)
			extent_op->update_key = 0;
		else
			extent_op->update_key = 1;
6888 6889
		extent_op->update_flags = 1;
		extent_op->is_data = 0;
6890
		extent_op->level = level;
6891

6892 6893
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
6894 6895
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
6896
					extent_op, 0);
6897
		BUG_ON(ret); /* -ENOMEM */
6898
	}
6899 6900
	return buf;
}
6901

6902 6903 6904 6905 6906 6907 6908 6909 6910
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
6911 6912
	int reada_slot;
	int reada_count;
6913
	int for_reloc;
6914 6915 6916 6917 6918
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

6919 6920 6921 6922
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
6923
{
6924 6925 6926
	u64 bytenr;
	u64 generation;
	u64 refs;
6927
	u64 flags;
6928
	u32 nritems;
6929 6930 6931
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
6932
	int ret;
6933 6934
	int slot;
	int nread = 0;
6935

6936 6937 6938 6939 6940 6941 6942 6943
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
6944

6945 6946 6947
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
6948

6949 6950 6951
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
6952

Chris Mason's avatar
Chris Mason committed
6953
		cond_resched();
6954 6955
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
Chris Mason's avatar
Chris Mason committed
6956

6957 6958
		if (slot == path->slots[wc->level])
			goto reada;
6959

6960 6961
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
6962 6963
			continue;

6964
		/* We don't lock the tree block, it's OK to be racy here */
6965 6966 6967
		ret = btrfs_lookup_extent_info(trans, root, bytenr,
					       wc->level - 1, 1, &refs,
					       &flags);
6968 6969 6970
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
6971 6972
		BUG_ON(refs == 0);

6973 6974 6975
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
6976

6977 6978 6979
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6980 6981 6982 6983 6984 6985 6986 6987
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
6988 6989 6990 6991
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6992
		}
6993 6994 6995 6996
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
6997
			break;
6998
		nread++;
6999
	}
7000
	wc->reada_slot = slot;
7001
}
7002

7003
/*
Liu Bo's avatar
Liu Bo committed
7004
 * helper to process tree block while walking down the tree.
7005 7006 7007 7008 7009
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
7010
 */
7011
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
7012
				   struct btrfs_root *root,
7013
				   struct btrfs_path *path,
7014
				   struct walk_control *wc, int lookup_info)
7015
{
7016 7017 7018
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
7019 7020
	int ret;

7021 7022 7023
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
7024

7025 7026 7027 7028
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
7029 7030 7031
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
7032 7033
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
7034
					       eb->start, level, 1,
7035 7036
					       &wc->refs[level],
					       &wc->flags[level]);
7037 7038 7039
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
7040 7041
		BUG_ON(wc->refs[level] == 0);
	}
7042

7043 7044 7045
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
7046

7047
		if (path->locks[level] && !wc->keep_locks) {
7048
			btrfs_tree_unlock_rw(eb, path->locks[level]);
7049 7050 7051 7052
			path->locks[level] = 0;
		}
		return 0;
	}
7053

7054 7055 7056
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
7057
		ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
7058
		BUG_ON(ret); /* -ENOMEM */
7059
		ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
7060
		BUG_ON(ret); /* -ENOMEM */
7061
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
7062 7063
						  eb->len, flag,
						  btrfs_header_level(eb), 0);
7064
		BUG_ON(ret); /* -ENOMEM */
7065 7066 7067 7068 7069 7070 7071 7072
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
7073
		btrfs_tree_unlock_rw(eb, path->locks[level]);
7074 7075 7076 7077 7078
		path->locks[level] = 0;
	}
	return 0;
}

7079
/*
Liu Bo's avatar
Liu Bo committed
7080
 * helper to process tree block pointer.
7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
7095
				 struct walk_control *wc, int *lookup_info)
7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
7115 7116
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
7117
		return 1;
7118
	}
7119 7120 7121 7122 7123 7124 7125

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
7126 7127
		if (!next)
			return -ENOMEM;
7128 7129 7130 7131 7132
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

7133
	ret = btrfs_lookup_extent_info(trans, root, bytenr, level - 1, 1,
7134 7135
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
7136 7137 7138 7139 7140
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

7141 7142 7143 7144
	if (unlikely(wc->refs[level - 1] == 0)) {
		btrfs_err(root->fs_info, "Missing references.");
		BUG();
	}
7145
	*lookup_info = 0;
7146

7147
	if (wc->stage == DROP_REFERENCE) {
7148
		if (wc->refs[level - 1] > 1) {
7149 7150 7151 7152
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
7166 7167 7168 7169
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
7170 7171
	}

7172
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
7173 7174 7175
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
7176
		*lookup_info = 1;
7177 7178 7179 7180 7181 7182
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
7183 7184
		if (!next || !extent_buffer_uptodate(next)) {
			free_extent_buffer(next);
7185
			return -EIO;
7186
		}
7187 7188 7189 7190 7191 7192 7193 7194
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
7195
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7196 7197 7198 7199 7200 7201 7202
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
7203 7204 7205 7206 7207 7208 7209 7210
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
7211

7212
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
7213
				root->root_key.objectid, level - 1, 0, 0);
7214
		BUG_ON(ret); /* -ENOMEM */
7215 7216 7217
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
7218
	*lookup_info = 1;
7219 7220 7221
	return 1;
}

7222
/*
Liu Bo's avatar
Liu Bo committed
7223
 * helper to process tree block while walking up the tree.
7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
7239
	int ret;
7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
7266
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7267 7268

			ret = btrfs_lookup_extent_info(trans, root,
7269
						       eb->start, level, 1,
7270 7271
						       &wc->refs[level],
						       &wc->flags[level]);
7272 7273
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
7274
				path->locks[level] = 0;
7275 7276
				return ret;
			}
7277 7278
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
7279
				btrfs_tree_unlock_rw(eb, path->locks[level]);
7280
				path->locks[level] = 0;
7281 7282
				return 1;
			}
7283
		}
7284
	}
7285

7286 7287
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
7288

7289 7290 7291
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
7292 7293
				ret = btrfs_dec_ref(trans, root, eb, 1,
						    wc->for_reloc);
7294
			else
7295 7296
				ret = btrfs_dec_ref(trans, root, eb, 0,
						    wc->for_reloc);
7297
			BUG_ON(ret); /* -ENOMEM */
7298 7299 7300 7301 7302 7303
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
7304
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
7321 7322
	}

7323
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
7324 7325 7326
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
7327
	return 0;
7328 7329 7330 7331 7332 7333 7334 7335
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
7336
	int lookup_info = 1;
7337 7338 7339
	int ret;

	while (level >= 0) {
7340
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
7341 7342 7343 7344 7345 7346
		if (ret > 0)
			break;

		if (level == 0)
			break;

7347 7348 7349 7350
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

7351
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
7352 7353 7354
		if (ret > 0) {
			path->slots[level]++;
			continue;
7355 7356
		} else if (ret < 0)
			return ret;
7357
		level = wc->level;
7358 7359 7360 7361
	}
	return 0;
}

7362
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
7363
				 struct btrfs_root *root,
7364
				 struct btrfs_path *path,
7365
				 struct walk_control *wc, int max_level)
7366
{
7367
	int level = wc->level;
7368
	int ret;
7369

7370 7371 7372 7373 7374 7375
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
7376 7377
			return 0;
		} else {
7378 7379 7380
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
7381

7382
			if (path->locks[level]) {
7383 7384
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
7385
				path->locks[level] = 0;
7386
			}
7387 7388 7389
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
7390 7391 7392 7393 7394
		}
	}
	return 1;
}

Chris Mason's avatar
Chris Mason committed
7395
/*
7396 7397 7398 7399 7400 7401 7402 7403 7404
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
7405 7406
 *
 * If called with for_reloc == 0, may exit early with -EAGAIN
Chris Mason's avatar
Chris Mason committed
7407
 */
7408
int btrfs_drop_snapshot(struct btrfs_root *root,
7409 7410
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
7411
{
7412
	struct btrfs_path *path;
7413 7414
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
7415
	struct btrfs_root_item *root_item = &root->root_item;
7416 7417 7418 7419 7420
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
7421

7422
	path = btrfs_alloc_path();
7423 7424 7425 7426
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
7427

7428
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
7429 7430
	if (!wc) {
		btrfs_free_path(path);
7431 7432
		err = -ENOMEM;
		goto out;
7433
	}
7434

7435
	trans = btrfs_start_transaction(tree_root, 0);
7436 7437 7438 7439
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
7440

7441 7442
	if (block_rsv)
		trans->block_rsv = block_rsv;
7443

7444
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
7445
		level = btrfs_header_level(root->node);
7446 7447
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
7448
		path->slots[level] = 0;
7449
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7450 7451
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
7452 7453
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
7454 7455 7456
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

7457
		level = root_item->drop_level;
7458
		BUG_ON(level == 0);
7459
		path->lowest_level = level;
7460 7461 7462 7463
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
7464
			goto out_end_trans;
7465
		}
7466
		WARN_ON(ret > 0);
7467

7468 7469 7470 7471
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
7472
		btrfs_unlock_up_safe(path, 0);
7473 7474 7475 7476 7477 7478 7479 7480

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
7481
						level, 1, &wc->refs[level],
7482
						&wc->flags[level]);
7483 7484 7485 7486
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
7487 7488 7489 7490 7491 7492 7493 7494 7495
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
7496
	}
7497 7498 7499 7500 7501 7502

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
7503
	wc->for_reloc = for_reloc;
7504
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7505

7506
	while (1) {
7507
		if (!for_reloc && btrfs_need_cleaner_sleep(root)) {
7508 7509 7510 7511 7512
			pr_debug("btrfs: drop snapshot early exit\n");
			err = -EAGAIN;
			goto out_end_trans;
		}

7513 7514 7515
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
7516
			break;
7517
		}
Chris Mason's avatar
Chris Mason committed
7518

7519 7520 7521
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
7522
			break;
7523 7524 7525 7526
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
7527 7528
			break;
		}
7529 7530 7531 7532 7533 7534 7535 7536 7537 7538

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
7539
		if (btrfs_should_end_transaction(trans, tree_root)) {
7540 7541 7542
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
7543 7544 7545 7546 7547
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
7548

7549
			btrfs_end_transaction_throttle(trans, tree_root);
7550
			trans = btrfs_start_transaction(tree_root, 0);
7551 7552 7553 7554
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
7555 7556
			if (block_rsv)
				trans->block_rsv = block_rsv;
7557
		}
7558
	}
7559
	btrfs_release_path(path);
7560 7561
	if (err)
		goto out_end_trans;
7562 7563

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
7564 7565 7566 7567
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
7568

7569
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
7570 7571
		ret = btrfs_find_root(tree_root, &root->root_key, path,
				      NULL, NULL);
7572 7573 7574 7575 7576
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
7577 7578 7579 7580 7581 7582 7583
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
7584 7585 7586 7587
		}
	}

	if (root->in_radix) {
7588
		btrfs_drop_and_free_fs_root(tree_root->fs_info, root);
7589 7590 7591
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
7592
		btrfs_put_fs_root(root);
7593
	}
7594
out_end_trans:
7595
	btrfs_end_transaction_throttle(trans, tree_root);
7596
out_free:
7597
	kfree(wc);
7598
	btrfs_free_path(path);
7599 7600 7601
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
7602
	return err;
7603
}
7604

7605 7606 7607 7608
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
7609
 * only used by relocation code
7610
 */
7611 7612 7613 7614 7615 7616
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
7617
	struct walk_control *wc;
7618 7619 7620 7621 7622
	int level;
	int parent_level;
	int ret = 0;
	int wret;

7623 7624
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

7625
	path = btrfs_alloc_path();
Tsutomu Itoh's avatar
Tsutomu Itoh committed
7626 7627
	if (!path)
		return -ENOMEM;
7628

7629
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
Tsutomu Itoh's avatar
Tsutomu Itoh committed
7630 7631 7632 7633
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
7634

7635
	btrfs_assert_tree_locked(parent);
7636 7637 7638 7639 7640
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

7641
	btrfs_assert_tree_locked(node);
7642 7643 7644
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
7645
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7646 7647 7648 7649 7650 7651 7652 7653

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
7654
	wc->for_reloc = 1;
7655
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7656 7657

	while (1) {
7658 7659
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
7660 7661
			ret = wret;
			break;
7662
		}
7663

7664
		wret = walk_up_tree(trans, root, path, wc, parent_level);
7665 7666 7667 7668 7669 7670
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

7671
	kfree(wc);
7672 7673 7674 7675
	btrfs_free_path(path);
	return ret;
}

7676 7677 7678
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
7679
	u64 stripped;
7680

7681 7682 7683 7684 7685 7686 7687
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
7688

7689 7690 7691 7692 7693 7694 7695 7696
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7697
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
David Woodhouse's avatar
David Woodhouse committed
7698
		BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
7699 7700
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

7725
		/* this is drive concat, leave it alone */
7726
	}
7727

7728 7729 7730
	return flags;
}

7731
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
7732
{
7733 7734
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
7735
	u64 min_allocable_bytes;
7736
	int ret = -ENOSPC;
7737

7738

7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

7751 7752
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
7753 7754 7755 7756 7757 7758

	if (cache->ro) {
		ret = 0;
		goto out;
	}

7759 7760 7761 7762
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7763 7764
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
7765 7766 7767 7768
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
7769
out:
7770 7771 7772 7773
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
7774

7775 7776
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
7777

7778 7779 7780 7781
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
7782

7783
	BUG_ON(cache->ro);
7784

Chris Mason's avatar
Chris Mason committed
7785
	trans = btrfs_join_transaction(root);
7786 7787
	if (IS_ERR(trans))
		return PTR_ERR(trans);
7788

7789
	alloc_flags = update_block_group_flags(root, cache->flags);
7790
	if (alloc_flags != cache->flags) {
7791
		ret = do_chunk_alloc(trans, root, alloc_flags,
7792 7793 7794 7795
				     CHUNK_ALLOC_FORCE);
		if (ret < 0)
			goto out;
	}
7796

7797
	ret = set_block_group_ro(cache, 0);
7798 7799 7800
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7801
	ret = do_chunk_alloc(trans, root, alloc_flags,
7802
			     CHUNK_ALLOC_FORCE);
7803 7804
	if (ret < 0)
		goto out;
7805
	ret = set_block_group_ro(cache, 0);
7806 7807 7808 7809
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
7810

7811 7812 7813 7814
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
7815
	return do_chunk_alloc(trans, root, alloc_flags,
7816
			      CHUNK_ALLOC_FORCE);
7817 7818
}

7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

7875
void btrfs_set_block_group_rw(struct btrfs_root *root,
7876
			      struct btrfs_block_group_cache *cache)
7877
{
7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
7891 7892
}

7893 7894 7895 7896 7897 7898 7899
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
7900
{
7901 7902 7903 7904
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
7905
	u64 min_free;
7906 7907
	u64 dev_min = 1;
	u64 dev_nr = 0;
7908
	u64 target;
7909
	int index;
7910 7911
	int full = 0;
	int ret = 0;
7912

7913
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
7914

7915 7916 7917
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
7918

7919 7920
	min_free = btrfs_block_group_used(&block_group->item);

7921
	/* no bytes used, we're good */
7922
	if (!min_free)
7923 7924
		goto out;

7925 7926
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
7927

7928
	full = space_info->full;
7929

7930 7931
	/*
	 * if this is the last block group we have in this space, we can't
7932 7933 7934 7935
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
7936
	 */
7937
	if ((space_info->total_bytes != block_group->key.offset) &&
7938 7939 7940
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
7941 7942
		spin_unlock(&space_info->lock);
		goto out;
7943
	}
7944
	spin_unlock(&space_info->lock);
7945

7946 7947 7948
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
7949 7950 7951
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
7952 7953
	 */
	ret = -1;
7954

7955 7956 7957 7958 7959 7960 7961 7962
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
7963 7964
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
7965
		index = __get_raid_index(extended_to_chunk(target));
7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

7977
	if (index == BTRFS_RAID_RAID10) {
7978
		dev_min = 4;
7979 7980
		/* Divide by 2 */
		min_free >>= 1;
7981
	} else if (index == BTRFS_RAID_RAID1) {
7982
		dev_min = 2;
7983
	} else if (index == BTRFS_RAID_DUP) {
7984 7985
		/* Multiply by 2 */
		min_free <<= 1;
7986
	} else if (index == BTRFS_RAID_RAID0) {
7987
		dev_min = fs_devices->rw_devices;
7988
		do_div(min_free, dev_min);
7989 7990
	}

7991 7992
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7993
		u64 dev_offset;
7994

7995 7996 7997 7998
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
7999 8000
		if (device->total_bytes > device->bytes_used + min_free &&
		    !device->is_tgtdev_for_dev_replace) {
8001
			ret = find_free_dev_extent(device, min_free,
8002
						   &dev_offset, NULL);
8003
			if (!ret)
8004 8005 8006
				dev_nr++;

			if (dev_nr >= dev_min)
8007
				break;
8008

8009
			ret = -1;
8010
		}
8011
	}
8012
	mutex_unlock(&root->fs_info->chunk_mutex);
8013
out:
8014
	btrfs_put_block_group(block_group);
8015 8016 8017
	return ret;
}

8018 8019
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
8020
{
8021
	int ret = 0;
8022 8023 8024
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
8025

8026 8027
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
8028 8029
		goto out;

8030
	while (1) {
8031
		slot = path->slots[0];
8032
		leaf = path->nodes[0];
8033 8034 8035 8036 8037
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
8038
				goto out;
8039
			break;
8040
		}
8041
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
8042

8043
		if (found_key.objectid >= key->objectid &&
8044 8045 8046 8047
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
8048
		path->slots[0]++;
8049
	}
8050
out:
8051
	return ret;
8052 8053
}

8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

8088 8089 8090
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
8091
	struct btrfs_space_info *space_info;
8092
	struct btrfs_caching_control *caching_ctl;
8093 8094
	struct rb_node *n;

8095 8096 8097 8098 8099 8100 8101 8102 8103
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

8104 8105 8106 8107 8108 8109
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Yan Zheng's avatar
Yan Zheng committed
8110 8111
		spin_unlock(&info->block_group_cache_lock);

8112
		down_write(&block_group->space_info->groups_sem);
8113
		list_del(&block_group->list);
8114
		up_write(&block_group->space_info->groups_sem);
8115

8116
		if (block_group->cached == BTRFS_CACHE_STARTED)
8117
			wait_block_group_cache_done(block_group);
8118

8119 8120 8121 8122 8123 8124 8125
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

8126
		btrfs_remove_free_space_cache(block_group);
8127
		btrfs_put_block_group(block_group);
Yan Zheng's avatar
Yan Zheng committed
8128 8129

		spin_lock(&info->block_group_cache_lock);
8130 8131
	}
	spin_unlock(&info->block_group_cache_lock);
8132 8133 8134 8135 8136 8137 8138 8139 8140

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

8141 8142
	release_global_block_rsv(info);

8143 8144 8145 8146
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
8147 8148 8149 8150 8151 8152 8153
		if (btrfs_test_opt(info->tree_root, ENOSPC_DEBUG)) {
			if (space_info->bytes_pinned > 0 ||
			    space_info->bytes_reserved > 0 ||
			    space_info->bytes_may_use > 0) {
				WARN_ON(1);
				dump_space_info(space_info, 0, 0);
			}
8154
		}
8155 8156 8157
		list_del(&space_info->list);
		kfree(space_info);
	}
8158 8159 8160
	return 0;
}

8161 8162 8163 8164 8165 8166 8167 8168 8169 8170
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

8171 8172 8173 8174 8175
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
8176
	struct btrfs_fs_info *info = root->fs_info;
8177
	struct btrfs_space_info *space_info;
8178 8179
	struct btrfs_key key;
	struct btrfs_key found_key;
8180
	struct extent_buffer *leaf;
8181 8182
	int need_clear = 0;
	u64 cache_gen;
8183

8184
	root = info->extent_root;
8185
	key.objectid = 0;
8186
	key.offset = 0;
8187 8188 8189 8190
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
8191
	path->reada = 1;
8192

8193
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
8194
	if (btrfs_test_opt(root, SPACE_CACHE) &&
8195
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
8196
		need_clear = 1;
8197 8198
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
8199

8200
	while (1) {
8201
		ret = find_first_block_group(root, path, &key);
8202 8203
		if (ret > 0)
			break;
8204 8205
		if (ret != 0)
			goto error;
8206 8207
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
8208
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
8209
		if (!cache) {
8210
			ret = -ENOMEM;
8211
			goto error;
8212
		}
8213 8214 8215 8216 8217 8218 8219
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
Chris Mason's avatar
Chris Mason committed
8220

8221
		atomic_set(&cache->count, 1);
8222
		spin_lock_init(&cache->lock);
8223
		cache->fs_info = info;
8224
		INIT_LIST_HEAD(&cache->list);
8225
		INIT_LIST_HEAD(&cache->cluster_list);
8226

8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
8238
			cache->disk_cache_state = BTRFS_DC_CLEAR;
8239 8240 8241
			if (btrfs_test_opt(root, SPACE_CACHE))
				cache->dirty = 1;
		}
8242

8243 8244 8245
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
8246
		memcpy(&cache->key, &found_key, sizeof(found_key));
8247

8248
		key.objectid = found_key.objectid + found_key.offset;
8249
		btrfs_release_path(path);
8250
		cache->flags = btrfs_block_group_flags(&cache->item);
8251
		cache->sectorsize = root->sectorsize;
David Woodhouse's avatar
David Woodhouse committed
8252 8253 8254
		cache->full_stripe_len = btrfs_full_stripe_len(root,
					       &root->fs_info->mapping_tree,
					       found_key.objectid);
8255 8256
		btrfs_init_free_space_ctl(cache);

8257 8258 8259 8260 8261
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272
		ret = exclude_super_stripes(root, cache);
		if (ret) {
			/*
			 * We may have excluded something, so call this just in
			 * case.
			 */
			free_excluded_extents(root, cache);
			kfree(cache->free_space_ctl);
			kfree(cache);
			goto error;
		}
8273

8274 8275 8276 8277 8278 8279 8280 8281
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
8282
			cache->last_byte_to_unpin = (u64)-1;
8283
			cache->cached = BTRFS_CACHE_FINISHED;
8284
			free_excluded_extents(root, cache);
8285
		} else if (btrfs_block_group_used(&cache->item) == 0) {
8286
			cache->last_byte_to_unpin = (u64)-1;
8287 8288 8289 8290 8291
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
8292
			free_excluded_extents(root, cache);
8293
		}
8294

8295 8296 8297 8298 8299 8300 8301
		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			btrfs_put_block_group(cache);
			goto error;
		}

8302 8303 8304
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
8305 8306 8307 8308 8309 8310 8311 8312 8313 8314
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			spin_lock(&info->block_group_cache_lock);
			rb_erase(&cache->cache_node,
				 &info->block_group_cache_tree);
			spin_unlock(&info->block_group_cache_lock);
			btrfs_put_block_group(cache);
			goto error;
		}

8315
		cache->space_info = space_info;
8316
		spin_lock(&cache->space_info->lock);
8317
		cache->space_info->bytes_readonly += cache->bytes_super;
8318 8319
		spin_unlock(&cache->space_info->lock);

8320
		__link_block_group(space_info, cache);
8321

8322
		set_avail_alloc_bits(root->fs_info, cache->flags);
Yan Zheng's avatar
Yan Zheng committed
8323
		if (btrfs_chunk_readonly(root, cache->key.objectid))
8324
			set_block_group_ro(cache, 1);
8325
	}
8326 8327 8328 8329 8330

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
David Woodhouse's avatar
David Woodhouse committed
8331 8332
		       BTRFS_BLOCK_GROUP_RAID5 |
		       BTRFS_BLOCK_GROUP_RAID6 |
8333 8334 8335 8336 8337 8338 8339
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
8340
			set_block_group_ro(cache, 1);
8341
		list_for_each_entry(cache, &space_info->block_groups[4], list)
8342
			set_block_group_ro(cache, 1);
8343
	}
8344 8345

	init_global_block_rsv(info);
8346 8347
	ret = 0;
error:
8348
	btrfs_free_path(path);
8349
	return ret;
8350
}
8351

8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_group_cache *block_group, *tmp;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
	struct btrfs_block_group_item item;
	struct btrfs_key key;
	int ret = 0;

	list_for_each_entry_safe(block_group, tmp, &trans->new_bgs,
				 new_bg_list) {
		list_del_init(&block_group->new_bg_list);

		if (ret)
			continue;

		spin_lock(&block_group->lock);
		memcpy(&item, &block_group->item, sizeof(item));
		memcpy(&key, &block_group->key, sizeof(key));
		spin_unlock(&block_group->lock);

		ret = btrfs_insert_item(trans, extent_root, &key, &item,
					sizeof(item));
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
	}
}

8380 8381
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
8382
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
8383 8384 8385 8386 8387 8388 8389 8390
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

8391
	root->fs_info->last_trans_log_full_commit = trans->transid;
8392

8393
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
8394 8395
	if (!cache)
		return -ENOMEM;
8396 8397 8398 8399 8400 8401
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
8402

8403
	cache->key.objectid = chunk_offset;
8404
	cache->key.offset = size;
8405
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
8406
	cache->sectorsize = root->sectorsize;
8407
	cache->fs_info = root->fs_info;
David Woodhouse's avatar
David Woodhouse committed
8408 8409 8410
	cache->full_stripe_len = btrfs_full_stripe_len(root,
					       &root->fs_info->mapping_tree,
					       chunk_offset);
8411

8412
	atomic_set(&cache->count, 1);
8413
	spin_lock_init(&cache->lock);
8414
	INIT_LIST_HEAD(&cache->list);
8415
	INIT_LIST_HEAD(&cache->cluster_list);
8416
	INIT_LIST_HEAD(&cache->new_bg_list);
8417

8418 8419
	btrfs_init_free_space_ctl(cache);

8420 8421 8422 8423 8424
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

8425
	cache->last_byte_to_unpin = (u64)-1;
8426
	cache->cached = BTRFS_CACHE_FINISHED;
8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437
	ret = exclude_super_stripes(root, cache);
	if (ret) {
		/*
		 * We may have excluded something, so call this just in
		 * case.
		 */
		free_excluded_extents(root, cache);
		kfree(cache->free_space_ctl);
		kfree(cache);
		return ret;
	}
8438

8439 8440 8441
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

8442 8443
	free_excluded_extents(root, cache);

8444 8445 8446 8447 8448 8449 8450
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		btrfs_put_block_group(cache);
		return ret;
	}

8451 8452
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
8453 8454 8455 8456 8457 8458 8459 8460 8461
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		spin_lock(&root->fs_info->block_group_cache_lock);
		rb_erase(&cache->cache_node,
			 &root->fs_info->block_group_cache_tree);
		spin_unlock(&root->fs_info->block_group_cache_lock);
		btrfs_put_block_group(cache);
		return ret;
	}
8462
	update_global_block_rsv(root->fs_info);
8463 8464

	spin_lock(&cache->space_info->lock);
8465
	cache->space_info->bytes_readonly += cache->bytes_super;
8466 8467
	spin_unlock(&cache->space_info->lock);

8468
	__link_block_group(cache->space_info, cache);
8469

8470
	list_add_tail(&cache->new_bg_list, &trans->new_bgs);
8471

8472
	set_avail_alloc_bits(extent_root->fs_info, type);
8473

8474 8475
	return 0;
}
8476

8477 8478
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
8479 8480
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
8481

8482
	write_seqlock(&fs_info->profiles_lock);
8483 8484 8485 8486 8487 8488
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
8489
	write_sequnlock(&fs_info->profiles_lock);
8490 8491
}

8492 8493 8494 8495 8496
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
8497
	struct btrfs_free_cluster *cluster;
8498
	struct btrfs_root *tree_root = root->fs_info->tree_root;
8499
	struct btrfs_key key;
8500
	struct inode *inode;
8501
	int ret;
8502
	int index;
Josef Bacik's avatar
Josef Bacik committed
8503
	int factor;
8504 8505 8506 8507 8508

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
8509
	BUG_ON(!block_group->ro);
8510

8511 8512 8513 8514 8515 8516
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

8517
	memcpy(&key, &block_group->key, sizeof(key));
8518
	index = get_block_group_index(block_group);
Josef Bacik's avatar
Josef Bacik committed
8519 8520 8521 8522 8523 8524
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
8525

8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

8541
	path = btrfs_alloc_path();
8542 8543 8544 8545
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
8546

8547
	inode = lookup_free_space_inode(tree_root, block_group, path);
8548
	if (!IS_ERR(inode)) {
8549
		ret = btrfs_orphan_add(trans, inode);
8550 8551 8552 8553
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
8566
		btrfs_add_delayed_iput(inode);
8567 8568 8569 8570 8571 8572 8573 8574 8575 8576
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
8577
		btrfs_release_path(path);
8578 8579 8580 8581
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
8582
		btrfs_release_path(path);
8583 8584
	}

8585
	spin_lock(&root->fs_info->block_group_cache_lock);
8586 8587
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
8588 8589 8590

	if (root->fs_info->first_logical_byte == block_group->key.objectid)
		root->fs_info->first_logical_byte = (u64)-1;
8591
	spin_unlock(&root->fs_info->block_group_cache_lock);
8592

8593
	down_write(&block_group->space_info->groups_sem);
8594 8595 8596 8597 8598
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
8599 8600
	if (list_empty(&block_group->space_info->block_groups[index]))
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
8601
	up_write(&block_group->space_info->groups_sem);
8602

8603
	if (block_group->cached == BTRFS_CACHE_STARTED)
8604
		wait_block_group_cache_done(block_group);
8605 8606 8607

	btrfs_remove_free_space_cache(block_group);

8608 8609 8610
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
Josef Bacik's avatar
Josef Bacik committed
8611
	block_group->space_info->disk_total -= block_group->key.offset * factor;
8612
	spin_unlock(&block_group->space_info->lock);
8613

8614 8615
	memcpy(&key, &block_group->key, sizeof(key));

8616
	btrfs_clear_space_info_full(root->fs_info);
8617

8618 8619
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
8632

8633 8634 8635
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
8636 8637 8638 8639
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
8640 8641
	int ret;

8642
	disk_super = fs_info->super_copy;
8643 8644
	if (!btrfs_super_root(disk_super))
		return 1;
8645

8646 8647 8648
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
8649

8650 8651
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8652
	if (ret)
8653
		goto out;
8654

8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
8668 8669 8670
	return ret;
}

8671 8672 8673 8674 8675 8676
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8677
			       u64 num_bytes, u64 *actual_bytes)
8678
{
8679
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
8680
}
8681 8682 8683 8684 8685 8686 8687 8688 8689

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
8690
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8691 8692
	int ret = 0;

8693 8694 8695 8696 8697 8698 8699
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
8713
				ret = cache_block_group(cache, 0);
8714 8715 8716 8717 8718 8719 8720 8721 8722
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
				ret = wait_block_group_cache_done(cache);
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}