inode.c 283 KB
Newer Older
Chris Mason's avatar
Chris Mason committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */

19
#include <linux/kernel.h>
20
#include <linux/bio.h>
Chris Mason's avatar
Chris Mason committed
21
#include <linux/buffer_head.h>
Sage Weil's avatar
Sage Weil committed
22
#include <linux/file.h>
Chris Mason's avatar
Chris Mason committed
23 24 25 26 27 28 29 30 31 32 33 34
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/time.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/backing-dev.h>
#include <linux/mpage.h>
#include <linux/swap.h>
#include <linux/writeback.h>
#include <linux/statfs.h>
#include <linux/compat.h>
35
#include <linux/bit_spinlock.h>
Josef Bacik's avatar
Josef Bacik committed
36
#include <linux/xattr.h>
Josef Bacik's avatar
Josef Bacik committed
37
#include <linux/posix_acl.h>
Yan Zheng's avatar
Yan Zheng committed
38
#include <linux/falloc.h>
39
#include <linux/slab.h>
40
#include <linux/ratelimit.h>
41
#include <linux/mount.h>
42
#include <linux/btrfs.h>
David Woodhouse's avatar
David Woodhouse committed
43
#include <linux/blkdev.h>
44
#include <linux/posix_acl_xattr.h>
45
#include <linux/uio.h>
Chris Mason's avatar
Chris Mason committed
46 47 48 49 50
#include "ctree.h"
#include "disk-io.h"
#include "transaction.h"
#include "btrfs_inode.h"
#include "print-tree.h"
51
#include "ordered-data.h"
52
#include "xattr.h"
53
#include "tree-log.h"
54
#include "volumes.h"
55
#include "compression.h"
56
#include "locking.h"
57
#include "free-space-cache.h"
58
#include "inode-map.h"
Liu Bo's avatar
Liu Bo committed
59
#include "backref.h"
60
#include "hash.h"
61
#include "props.h"
62
#include "qgroup.h"
63
#include "dedupe.h"
Chris Mason's avatar
Chris Mason committed
64 65

struct btrfs_iget_args {
66
	struct btrfs_key *location;
Chris Mason's avatar
Chris Mason committed
67 68 69
	struct btrfs_root *root;
};

70 71 72 73 74 75 76
struct btrfs_dio_data {
	u64 outstanding_extents;
	u64 reserve;
	u64 unsubmitted_oe_range_start;
	u64 unsubmitted_oe_range_end;
};

77 78 79 80 81
static const struct inode_operations btrfs_dir_inode_operations;
static const struct inode_operations btrfs_symlink_inode_operations;
static const struct inode_operations btrfs_dir_ro_inode_operations;
static const struct inode_operations btrfs_special_inode_operations;
static const struct inode_operations btrfs_file_inode_operations;
82 83
static const struct address_space_operations btrfs_aops;
static const struct address_space_operations btrfs_symlink_aops;
84
static const struct file_operations btrfs_dir_file_operations;
85
static const struct extent_io_ops btrfs_extent_io_ops;
Chris Mason's avatar
Chris Mason committed
86 87 88 89 90

static struct kmem_cache *btrfs_inode_cachep;
struct kmem_cache *btrfs_trans_handle_cachep;
struct kmem_cache *btrfs_transaction_cachep;
struct kmem_cache *btrfs_path_cachep;
91
struct kmem_cache *btrfs_free_space_cachep;
Chris Mason's avatar
Chris Mason committed
92 93

#define S_SHIFT 12
94
static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
Chris Mason's avatar
Chris Mason committed
95 96 97 98 99 100 101 102 103
	[S_IFREG >> S_SHIFT]	= BTRFS_FT_REG_FILE,
	[S_IFDIR >> S_SHIFT]	= BTRFS_FT_DIR,
	[S_IFCHR >> S_SHIFT]	= BTRFS_FT_CHRDEV,
	[S_IFBLK >> S_SHIFT]	= BTRFS_FT_BLKDEV,
	[S_IFIFO >> S_SHIFT]	= BTRFS_FT_FIFO,
	[S_IFSOCK >> S_SHIFT]	= BTRFS_FT_SOCK,
	[S_IFLNK >> S_SHIFT]	= BTRFS_FT_SYMLINK,
};

104
static int btrfs_setsize(struct inode *inode, struct iattr *attr);
105
static int btrfs_truncate(struct inode *inode);
106
static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
107 108
static noinline int cow_file_range(struct inode *inode,
				   struct page *locked_page,
109 110 111
				   u64 start, u64 end, u64 delalloc_end,
				   int *page_started, unsigned long *nr_written,
				   int unlock, struct btrfs_dedupe_hash *hash);
112 113 114
static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
					   u64 len, u64 orig_start,
					   u64 block_start, u64 block_len,
Josef Bacik's avatar
Josef Bacik committed
115 116
					   u64 orig_block_len, u64 ram_bytes,
					   int type);
117

118
static int btrfs_dirty_inode(struct inode *inode);
119

120 121 122 123 124 125 126
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
void btrfs_test_inode_set_ops(struct inode *inode)
{
	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
}
#endif

127
static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
128 129
				     struct inode *inode,  struct inode *dir,
				     const struct qstr *qstr)
Jim Owens's avatar
Jim Owens committed
130 131 132
{
	int err;

133
	err = btrfs_init_acl(trans, inode, dir);
Jim Owens's avatar
Jim Owens committed
134
	if (!err)
135
		err = btrfs_xattr_security_init(trans, inode, dir, qstr);
Jim Owens's avatar
Jim Owens committed
136 137 138
	return err;
}

139 140 141 142 143
/*
 * this does all the hard work for inserting an inline extent into
 * the btree.  The caller should have done a btrfs_drop_extents so that
 * no overlapping inline items exist in the btree
 */
144
static int insert_inline_extent(struct btrfs_trans_handle *trans,
145
				struct btrfs_path *path, int extent_inserted,
146 147
				struct btrfs_root *root, struct inode *inode,
				u64 start, size_t size, size_t compressed_size,
148
				int compress_type,
149 150 151 152 153 154 155 156 157 158 159 160
				struct page **compressed_pages)
{
	struct extent_buffer *leaf;
	struct page *page = NULL;
	char *kaddr;
	unsigned long ptr;
	struct btrfs_file_extent_item *ei;
	int err = 0;
	int ret;
	size_t cur_size = size;
	unsigned long offset;

161
	if (compressed_size && compressed_pages)
162 163
		cur_size = compressed_size;

164
	inode_add_bytes(inode, size);
165

166 167 168
	if (!extent_inserted) {
		struct btrfs_key key;
		size_t datasize;
169

170 171
		key.objectid = btrfs_ino(inode);
		key.offset = start;
172
		key.type = BTRFS_EXTENT_DATA_KEY;
173

174 175 176 177 178 179 180 181
		datasize = btrfs_file_extent_calc_inline_size(cur_size);
		path->leave_spinning = 1;
		ret = btrfs_insert_empty_item(trans, root, path, &key,
					      datasize);
		if (ret) {
			err = ret;
			goto fail;
		}
182 183 184 185 186 187 188 189 190 191 192
	}
	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0],
			    struct btrfs_file_extent_item);
	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
	btrfs_set_file_extent_encryption(leaf, ei, 0);
	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
	ptr = btrfs_file_extent_inline_start(ei);

193
	if (compress_type != BTRFS_COMPRESS_NONE) {
194 195
		struct page *cpage;
		int i = 0;
196
		while (compressed_size > 0) {
197
			cpage = compressed_pages[i];
198
			cur_size = min_t(unsigned long, compressed_size,
199
				       PAGE_SIZE);
200

201
			kaddr = kmap_atomic(cpage);
202
			write_extent_buffer(leaf, kaddr, ptr, cur_size);
203
			kunmap_atomic(kaddr);
204 205 206 207 208 209

			i++;
			ptr += cur_size;
			compressed_size -= cur_size;
		}
		btrfs_set_file_extent_compression(leaf, ei,
210
						  compress_type);
211 212
	} else {
		page = find_get_page(inode->i_mapping,
213
				     start >> PAGE_SHIFT);
214
		btrfs_set_file_extent_compression(leaf, ei, 0);
215
		kaddr = kmap_atomic(page);
216
		offset = start & (PAGE_SIZE - 1);
217
		write_extent_buffer(leaf, kaddr + offset, ptr, size);
218
		kunmap_atomic(kaddr);
219
		put_page(page);
220 221
	}
	btrfs_mark_buffer_dirty(leaf);
222
	btrfs_release_path(path);
223

224 225 226 227 228 229 230 231 232
	/*
	 * we're an inline extent, so nobody can
	 * extend the file past i_size without locking
	 * a page we already have locked.
	 *
	 * We must do any isize and inode updates
	 * before we unlock the pages.  Otherwise we
	 * could end up racing with unlink.
	 */
233
	BTRFS_I(inode)->disk_i_size = inode->i_size;
234
	ret = btrfs_update_inode(trans, root, inode);
235

236
	return ret;
237 238 239 240 241 242 243 244 245 246
fail:
	return err;
}


/*
 * conditionally insert an inline extent into the file.  This
 * does the checks required to make sure the data is small enough
 * to fit as an inline extent.
 */
247 248 249 250 251
static noinline int cow_file_range_inline(struct btrfs_root *root,
					  struct inode *inode, u64 start,
					  u64 end, size_t compressed_size,
					  int compress_type,
					  struct page **compressed_pages)
252
{
253
	struct btrfs_trans_handle *trans;
254 255 256
	u64 isize = i_size_read(inode);
	u64 actual_end = min(end + 1, isize);
	u64 inline_len = actual_end - start;
257
	u64 aligned_end = ALIGN(end, root->sectorsize);
258 259
	u64 data_len = inline_len;
	int ret;
260 261 262
	struct btrfs_path *path;
	int extent_inserted = 0;
	u32 extent_item_size;
263 264 265 266 267

	if (compressed_size)
		data_len = compressed_size;

	if (start > 0 ||
268
	    actual_end > root->sectorsize ||
269
	    data_len > BTRFS_MAX_INLINE_DATA_SIZE(root) ||
270 271 272 273 274 275 276
	    (!compressed_size &&
	    (actual_end & (root->sectorsize - 1)) == 0) ||
	    end + 1 < isize ||
	    data_len > root->fs_info->max_inline) {
		return 1;
	}

277 278 279 280
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

281
	trans = btrfs_join_transaction(root);
282 283
	if (IS_ERR(trans)) {
		btrfs_free_path(path);
284
		return PTR_ERR(trans);
285
	}
286 287
	trans->block_rsv = &root->fs_info->delalloc_block_rsv;

288 289 290 291 292 293 294 295 296 297
	if (compressed_size && compressed_pages)
		extent_item_size = btrfs_file_extent_calc_inline_size(
		   compressed_size);
	else
		extent_item_size = btrfs_file_extent_calc_inline_size(
		    inline_len);

	ret = __btrfs_drop_extents(trans, root, inode, path,
				   start, aligned_end, NULL,
				   1, 1, extent_item_size, &extent_inserted);
298
	if (ret) {
299
		btrfs_abort_transaction(trans, ret);
300 301
		goto out;
	}
302 303 304

	if (isize > actual_end)
		inline_len = min_t(u64, isize, actual_end);
305 306
	ret = insert_inline_extent(trans, path, extent_inserted,
				   root, inode, start,
307
				   inline_len, compressed_size,
308
				   compress_type, compressed_pages);
309
	if (ret && ret != -ENOSPC) {
310
		btrfs_abort_transaction(trans, ret);
311
		goto out;
312
	} else if (ret == -ENOSPC) {
313 314
		ret = 1;
		goto out;
315
	}
316

317
	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
318
	btrfs_delalloc_release_metadata(inode, end + 1 - start);
319
	btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
320
out:
321 322 323 324 325 326
	/*
	 * Don't forget to free the reserved space, as for inlined extent
	 * it won't count as data extent, free them directly here.
	 * And at reserve time, it's always aligned to page size, so
	 * just free one page here.
	 */
327
	btrfs_qgroup_free_data(inode, 0, PAGE_SIZE);
328
	btrfs_free_path(path);
329 330
	btrfs_end_transaction(trans, root);
	return ret;
331 332
}

333 334 335 336 337 338
struct async_extent {
	u64 start;
	u64 ram_size;
	u64 compressed_size;
	struct page **pages;
	unsigned long nr_pages;
339
	int compress_type;
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
	struct list_head list;
};

struct async_cow {
	struct inode *inode;
	struct btrfs_root *root;
	struct page *locked_page;
	u64 start;
	u64 end;
	struct list_head extents;
	struct btrfs_work work;
};

static noinline int add_async_extent(struct async_cow *cow,
				     u64 start, u64 ram_size,
				     u64 compressed_size,
				     struct page **pages,
357 358
				     unsigned long nr_pages,
				     int compress_type)
359 360 361 362
{
	struct async_extent *async_extent;

	async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
363
	BUG_ON(!async_extent); /* -ENOMEM */
364 365 366 367 368
	async_extent->start = start;
	async_extent->ram_size = ram_size;
	async_extent->compressed_size = compressed_size;
	async_extent->pages = pages;
	async_extent->nr_pages = nr_pages;
369
	async_extent->compress_type = compress_type;
370 371 372 373
	list_add_tail(&async_extent->list, &cow->extents);
	return 0;
}

374 375 376 377 378
static inline int inode_need_compress(struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;

	/* force compress */
379
	if (btrfs_test_opt(root->fs_info, FORCE_COMPRESS))
380 381 382 383
		return 1;
	/* bad compression ratios */
	if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
		return 0;
384
	if (btrfs_test_opt(root->fs_info, COMPRESS) ||
385 386 387 388 389 390
	    BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
	    BTRFS_I(inode)->force_compress)
		return 1;
	return 0;
}

391
/*
392 393 394
 * we create compressed extents in two phases.  The first
 * phase compresses a range of pages that have already been
 * locked (both pages and state bits are locked).
395
 *
396 397 398 399 400
 * This is done inside an ordered work queue, and the compression
 * is spread across many cpus.  The actual IO submission is step
 * two, and the ordered work queue takes care of making sure that
 * happens in the same order things were put onto the queue by
 * writepages and friends.
401
 *
402 403 404
 * If this code finds it can't get good compression, it puts an
 * entry onto the work queue to write the uncompressed bytes.  This
 * makes sure that both compressed inodes and uncompressed inodes
405 406
 * are written in the same order that the flusher thread sent them
 * down.
407
 */
408
static noinline void compress_file_range(struct inode *inode,
409 410 411 412
					struct page *locked_page,
					u64 start, u64 end,
					struct async_cow *async_cow,
					int *num_added)
413 414
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
415 416
	u64 num_bytes;
	u64 blocksize = root->sectorsize;
417
	u64 actual_end;
418
	u64 isize = i_size_read(inode);
419
	int ret = 0;
420 421 422 423 424
	struct page **pages = NULL;
	unsigned long nr_pages;
	unsigned long nr_pages_ret = 0;
	unsigned long total_compressed = 0;
	unsigned long total_in = 0;
425 426
	unsigned long max_compressed = SZ_128K;
	unsigned long max_uncompressed = SZ_128K;
427 428
	int i;
	int will_compress;
429
	int compress_type = root->fs_info->compress_type;
430
	int redirty = 0;
431

432
	/* if this is a small write inside eof, kick off a defrag */
433
	if ((end - start + 1) < SZ_16K &&
434
	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
435 436
		btrfs_add_inode_defrag(NULL, inode);

437
	actual_end = min_t(u64, isize, end + 1);
438 439
again:
	will_compress = 0;
440 441
	nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
	nr_pages = min_t(unsigned long, nr_pages, SZ_128K / PAGE_SIZE);
442

443 444 445 446 447 448 449 450 451 452 453 454 455
	/*
	 * we don't want to send crud past the end of i_size through
	 * compression, that's just a waste of CPU time.  So, if the
	 * end of the file is before the start of our current
	 * requested range of bytes, we bail out to the uncompressed
	 * cleanup code that can deal with all of this.
	 *
	 * It isn't really the fastest way to fix things, but this is a
	 * very uncommon corner.
	 */
	if (actual_end <= start)
		goto cleanup_and_bail_uncompressed;

456 457
	total_compressed = actual_end - start;

458 459
	/*
	 * skip compression for a small file range(<=blocksize) that
460
	 * isn't an inline extent, since it doesn't save disk space at all.
461 462 463 464 465
	 */
	if (total_compressed <= blocksize &&
	   (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
		goto cleanup_and_bail_uncompressed;

466 467
	/* we want to make sure that amount of ram required to uncompress
	 * an extent is reasonable, so we limit the total size in ram
468 469 470 471 472 473 474
	 * of a compressed extent to 128k.  This is a crucial number
	 * because it also controls how easily we can spread reads across
	 * cpus for decompression.
	 *
	 * We also want to make sure the amount of IO required to do
	 * a random read is reasonably small, so we limit the size of
	 * a compressed extent to 128k.
475 476
	 */
	total_compressed = min(total_compressed, max_uncompressed);
477
	num_bytes = ALIGN(end - start + 1, blocksize);
478
	num_bytes = max(blocksize,  num_bytes);
479 480
	total_in = 0;
	ret = 0;
481

482 483 484 485
	/*
	 * we do compression for mount -o compress and when the
	 * inode has not been flagged as nocompress.  This flag can
	 * change at any time if we discover bad compression ratios.
486
	 */
487
	if (inode_need_compress(inode)) {
488
		WARN_ON(pages);
489
		pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
490 491 492 493
		if (!pages) {
			/* just bail out to the uncompressed code */
			goto cont;
		}
494

495 496 497
		if (BTRFS_I(inode)->force_compress)
			compress_type = BTRFS_I(inode)->force_compress;

498 499 500 501 502 503 504 505 506 507 508
		/*
		 * we need to call clear_page_dirty_for_io on each
		 * page in the range.  Otherwise applications with the file
		 * mmap'd can wander in and change the page contents while
		 * we are compressing them.
		 *
		 * If the compression fails for any reason, we set the pages
		 * dirty again later on.
		 */
		extent_range_clear_dirty_for_io(inode, start, end);
		redirty = 1;
509 510 511 512 513 514 515
		ret = btrfs_compress_pages(compress_type,
					   inode->i_mapping, start,
					   total_compressed, pages,
					   nr_pages, &nr_pages_ret,
					   &total_in,
					   &total_compressed,
					   max_compressed);
516 517 518

		if (!ret) {
			unsigned long offset = total_compressed &
519
				(PAGE_SIZE - 1);
520 521 522 523 524 525 526
			struct page *page = pages[nr_pages_ret - 1];
			char *kaddr;

			/* zero the tail end of the last page, we might be
			 * sending it down to disk
			 */
			if (offset) {
527
				kaddr = kmap_atomic(page);
528
				memset(kaddr + offset, 0,
529
				       PAGE_SIZE - offset);
530
				kunmap_atomic(kaddr);
531 532 533 534
			}
			will_compress = 1;
		}
	}
535
cont:
536 537
	if (start == 0) {
		/* lets try to make an inline extent */
538
		if (ret || total_in < (actual_end - start)) {
539
			/* we didn't compress the entire range, try
540
			 * to make an uncompressed inline extent.
541
			 */
542 543
			ret = cow_file_range_inline(root, inode, start, end,
						    0, 0, NULL);
544
		} else {
545
			/* try making a compressed inline extent */
546
			ret = cow_file_range_inline(root, inode, start, end,
547 548
						    total_compressed,
						    compress_type, pages);
549
		}
550
		if (ret <= 0) {
551 552
			unsigned long clear_flags = EXTENT_DELALLOC |
				EXTENT_DEFRAG;
553 554
			unsigned long page_error_op;

555
			clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0;
556
			page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
557

558
			/*
559 560 561
			 * inline extent creation worked or returned error,
			 * we don't need to create any more async work items.
			 * Unlock and free up our temp pages.
562
			 */
563 564 565
			extent_clear_unlock_delalloc(inode, start, end, end,
						     NULL, clear_flags,
						     PAGE_UNLOCK |
566 567
						     PAGE_CLEAR_DIRTY |
						     PAGE_SET_WRITEBACK |
568
						     page_error_op |
569
						     PAGE_END_WRITEBACK);
570 571
			btrfs_free_reserved_data_space_noquota(inode, start,
						end - start + 1);
572 573 574 575 576 577 578 579 580 581
			goto free_pages_out;
		}
	}

	if (will_compress) {
		/*
		 * we aren't doing an inline extent round the compressed size
		 * up to a block size boundary so the allocator does sane
		 * things
		 */
582
		total_compressed = ALIGN(total_compressed, blocksize);
583 584 585 586 587

		/*
		 * one last check to make sure the compression is really a
		 * win, compare the page count read with the blocks on disk
		 */
588
		total_in = ALIGN(total_in, PAGE_SIZE);
589 590 591 592
		if (total_compressed >= total_in) {
			will_compress = 0;
		} else {
			num_bytes = total_in;
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
			*num_added += 1;

			/*
			 * The async work queues will take care of doing actual
			 * allocation on disk for these compressed pages, and
			 * will submit them to the elevator.
			 */
			add_async_extent(async_cow, start, num_bytes,
					total_compressed, pages, nr_pages_ret,
					compress_type);

			if (start + num_bytes < end) {
				start += num_bytes;
				pages = NULL;
				cond_resched();
				goto again;
			}
			return;
611 612
		}
	}
613
	if (pages) {
614 615 616 617 618
		/*
		 * the compression code ran but failed to make things smaller,
		 * free any pages it allocated and our page pointer array
		 */
		for (i = 0; i < nr_pages_ret; i++) {
619
			WARN_ON(pages[i]->mapping);
620
			put_page(pages[i]);
621 622 623 624 625 626 627
		}
		kfree(pages);
		pages = NULL;
		total_compressed = 0;
		nr_pages_ret = 0;

		/* flag the file so we don't compress in the future */
628
		if (!btrfs_test_opt(root->fs_info, FORCE_COMPRESS) &&
629
		    !(BTRFS_I(inode)->force_compress)) {
630
			BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
631
		}
632
	}
633
cleanup_and_bail_uncompressed:
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649
	/*
	 * No compression, but we still need to write the pages in the file
	 * we've been given so far.  redirty the locked page if it corresponds
	 * to our extent and set things up for the async work queue to run
	 * cow_file_range to do the normal delalloc dance.
	 */
	if (page_offset(locked_page) >= start &&
	    page_offset(locked_page) <= end)
		__set_page_dirty_nobuffers(locked_page);
		/* unlocked later on in the async handlers */

	if (redirty)
		extent_range_redirty_for_io(inode, start, end);
	add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
			 BTRFS_COMPRESS_NONE);
	*num_added += 1;
650

651
	return;
652 653 654 655

free_pages_out:
	for (i = 0; i < nr_pages_ret; i++) {
		WARN_ON(pages[i]->mapping);
656
		put_page(pages[i]);
657
	}
658
	kfree(pages);
659 660
}

661 662 663 664 665 666 667 668 669
static void free_async_extent_pages(struct async_extent *async_extent)
{
	int i;

	if (!async_extent->pages)
		return;

	for (i = 0; i < async_extent->nr_pages; i++) {
		WARN_ON(async_extent->pages[i]->mapping);
670
		put_page(async_extent->pages[i]);
671 672 673 674
	}
	kfree(async_extent->pages);
	async_extent->nr_pages = 0;
	async_extent->pages = NULL;
675 676 677 678 679 680 681 682
}

/*
 * phase two of compressed writeback.  This is the ordered portion
 * of the code, which only gets called in the order the work was
 * queued.  We walk all the async extents created by compress_file_range
 * and send them down to the disk.
 */
683
static noinline void submit_compressed_extents(struct inode *inode,
684 685 686 687 688 689 690 691 692
					      struct async_cow *async_cow)
{
	struct async_extent *async_extent;
	u64 alloc_hint = 0;
	struct btrfs_key ins;
	struct extent_map *em;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
	struct extent_io_tree *io_tree;
693
	int ret = 0;
694

695
again:
696
	while (!list_empty(&async_cow->extents)) {
697 698 699
		async_extent = list_entry(async_cow->extents.next,
					  struct async_extent, list);
		list_del(&async_extent->list);
700

701 702
		io_tree = &BTRFS_I(inode)->io_tree;

703
retry:
704 705 706 707 708 709
		/* did the compression code fall back to uncompressed IO? */
		if (!async_extent->pages) {
			int page_started = 0;
			unsigned long nr_written = 0;

			lock_extent(io_tree, async_extent->start,
710
					 async_extent->start +
711
					 async_extent->ram_size - 1);
712 713

			/* allocate blocks */
714 715 716 717
			ret = cow_file_range(inode, async_cow->locked_page,
					     async_extent->start,
					     async_extent->start +
					     async_extent->ram_size - 1,
718 719 720 721
					     async_extent->start +
					     async_extent->ram_size - 1,
					     &page_started, &nr_written, 0,
					     NULL);
722

723 724
			/* JDM XXX */

725 726 727 728 729 730
			/*
			 * if page_started, cow_file_range inserted an
			 * inline extent and took care of all the unlocking
			 * and IO for us.  Otherwise, we need to submit
			 * all those pages down to the drive.
			 */
731
			if (!page_started && !ret)
732 733
				extent_write_locked_range(io_tree,
						  inode, async_extent->start,
734
						  async_extent->start +
735 736 737
						  async_extent->ram_size - 1,
						  btrfs_get_extent,
						  WB_SYNC_ALL);
738 739
			else if (ret)
				unlock_page(async_cow->locked_page);
740 741 742 743 744 745
			kfree(async_extent);
			cond_resched();
			continue;
		}

		lock_extent(io_tree, async_extent->start,
746
			    async_extent->start + async_extent->ram_size - 1);
747

748
		ret = btrfs_reserve_extent(root, async_extent->ram_size,
749 750
					   async_extent->compressed_size,
					   async_extent->compressed_size,
751
					   0, alloc_hint, &ins, 1, 1);
752
		if (ret) {
753
			free_async_extent_pages(async_extent);
754

755 756 757 758
			if (ret == -ENOSPC) {
				unlock_extent(io_tree, async_extent->start,
					      async_extent->start +
					      async_extent->ram_size - 1);
759 760 761 762 763 764 765 766 767 768 769 770

				/*
				 * we need to redirty the pages if we decide to
				 * fallback to uncompressed IO, otherwise we
				 * will not submit these pages down to lower
				 * layers.
				 */
				extent_range_redirty_for_io(inode,
						async_extent->start,
						async_extent->start +
						async_extent->ram_size - 1);

771
				goto retry;
772
			}
773
			goto out_free;
774
		}
775 776 777 778 779 780 781 782
		/*
		 * here we're doing allocation and writeback of the
		 * compressed pages
		 */
		btrfs_drop_extent_cache(inode, async_extent->start,
					async_extent->start +
					async_extent->ram_size - 1, 0);

783
		em = alloc_extent_map();
784 785
		if (!em) {
			ret = -ENOMEM;
786
			goto out_free_reserve;
787
		}
788 789
		em->start = async_extent->start;
		em->len = async_extent->ram_size;
790
		em->orig_start = em->start;
791 792
		em->mod_start = em->start;
		em->mod_len = em->len;
793

794 795
		em->block_start = ins.objectid;
		em->block_len = ins.offset;
796
		em->orig_block_len = ins.offset;
Josef Bacik's avatar
Josef Bacik committed
797
		em->ram_bytes = async_extent->ram_size;
798
		em->bdev = root->fs_info->fs_devices->latest_bdev;
799
		em->compress_type = async_extent->compress_type;
800 801
		set_bit(EXTENT_FLAG_PINNED, &em->flags);
		set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
802
		em->generation = -1;
803

804
		while (1) {
805
			write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
806
			ret = add_extent_mapping(em_tree, em, 1);
807
			write_unlock(&em_tree->lock);
808 809 810 811 812 813 814 815 816
			if (ret != -EEXIST) {
				free_extent_map(em);
				break;
			}
			btrfs_drop_extent_cache(inode, async_extent->start,
						async_extent->start +
						async_extent->ram_size - 1, 0);
		}

817 818 819
		if (ret)
			goto out_free_reserve;

820 821 822 823 824 825 826
		ret = btrfs_add_ordered_extent_compress(inode,
						async_extent->start,
						ins.objectid,
						async_extent->ram_size,
						ins.offset,
						BTRFS_ORDERED_COMPRESSED,
						async_extent->compress_type);
827 828 829 830
		if (ret) {
			btrfs_drop_extent_cache(inode, async_extent->start,
						async_extent->start +
						async_extent->ram_size - 1, 0);
831
			goto out_free_reserve;
832
		}
833
		btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);
834 835 836 837

		/*
		 * clear dirty, set writeback and unlock the pages.
		 */
838
		extent_clear_unlock_delalloc(inode, async_extent->start,
839 840
				async_extent->start +
				async_extent->ram_size - 1,
841 842
				async_extent->start +
				async_extent->ram_size - 1,
843 844
				NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
				PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
845
				PAGE_SET_WRITEBACK);
846
		ret = btrfs_submit_compressed_write(inode,
847 848 849 850 851
				    async_extent->start,
				    async_extent->ram_size,
				    ins.objectid,
				    ins.offset, async_extent->pages,
				    async_extent->nr_pages);
852 853 854 855 856 857 858 859 860 861
		if (ret) {
			struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
			struct page *p = async_extent->pages[0];
			const u64 start = async_extent->start;
			const u64 end = start + async_extent->ram_size - 1;

			p->mapping = inode->i_mapping;
			tree->ops->writepage_end_io_hook(p, start, end,
							 NULL, 0);
			p->mapping = NULL;
862 863
			extent_clear_unlock_delalloc(inode, start, end, end,
						     NULL, 0,
864 865
						     PAGE_END_WRITEBACK |
						     PAGE_SET_ERROR);
866
			free_async_extent_pages(async_extent);
867
		}
868 869 870 871
		alloc_hint = ins.objectid + ins.offset;
		kfree(async_extent);
		cond_resched();
	}
872
	return;
873
out_free_reserve:
874
	btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);
875
	btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
876
out_free:
877
	extent_clear_unlock_delalloc(inode, async_extent->start,
878 879
				     async_extent->start +
				     async_extent->ram_size - 1,
880 881
				     async_extent->start +
				     async_extent->ram_size - 1,
882
				     NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
883 884
				     EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
885 886
				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
				     PAGE_SET_ERROR);
887
	free_async_extent_pages(async_extent);
888
	kfree(async_extent);
889
	goto again;
890 891
}

892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
				      u64 num_bytes)
{
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
	struct extent_map *em;
	u64 alloc_hint = 0;

	read_lock(&em_tree->lock);
	em = search_extent_mapping(em_tree, start, num_bytes);
	if (em) {
		/*
		 * if block start isn't an actual block number then find the
		 * first block in this inode and use that as a hint.  If that
		 * block is also bogus then just don't worry about it.
		 */
		if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
			free_extent_map(em);
			em = search_extent_mapping(em_tree, 0, 0);
			if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
				alloc_hint = em->block_start;
			if (em)
				free_extent_map(em);
		} else {
			alloc_hint = em->block_start;
			free_extent_map(em);
		}
	}
	read_unlock(&em_tree->lock);

	return alloc_hint;
}

924 925 926 927 928 929 930 931 932 933 934 935 936
/*
 * when extent_io.c finds a delayed allocation range in the file,
 * the call backs end up in this code.  The basic idea is to
 * allocate extents on disk for the range, and create ordered data structs
 * in ram to track those extents.
 *
 * locked_page is the page that writepage had locked already.  We use
 * it to make sure we don't do extra locks or unlocks.
 *
 * *page_started is set to one if we unlock locked_page and do everything
 * required to start IO on it.  It may be clean and already done with
 * IO when we return.
 */
937 938
static noinline int cow_file_range(struct inode *inode,
				   struct page *locked_page,
939 940 941
				   u64 start, u64 end, u64 delalloc_end,
				   int *page_started, unsigned long *nr_written,
				   int unlock, struct btrfs_dedupe_hash *hash)
942
{
943
	struct btrfs_root *root = BTRFS_I(inode)->root;
944 945 946 947 948 949 950 951 952 953 954
	u64 alloc_hint = 0;
	u64 num_bytes;
	unsigned long ram_size;
	u64 disk_num_bytes;
	u64 cur_alloc_size;
	u64 blocksize = root->sectorsize;
	struct btrfs_key ins;
	struct extent_map *em;
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
	int ret = 0;

955 956
	if (btrfs_is_free_space_inode(inode)) {
		WARN_ON_ONCE(1);
957 958
		ret = -EINVAL;
		goto out_unlock;
959
	}
960

961
	num_bytes = ALIGN(end - start + 1, blocksize);
962 963 964
	num_bytes = max(blocksize,  num_bytes);
	disk_num_bytes = num_bytes;

965
	/* if this is a small write inside eof, kick off defrag */
966
	if (num_bytes < SZ_64K &&
967
	    (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
968
		btrfs_add_inode_defrag(NULL, inode);
969

970 971
	if (start == 0) {
		/* lets try to make an inline extent */
972 973
		ret = cow_file_range_inline(root, inode, start, end, 0, 0,
					    NULL);
974
		if (ret == 0) {
975 976
			extent_clear_unlock_delalloc(inode, start, end,
				     delalloc_end, NULL,
977
				     EXTENT_LOCKED | EXTENT_DELALLOC |
978
				     EXTENT_DEFRAG, PAGE_UNLOCK |
979 980
				     PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
				     PAGE_END_WRITEBACK);
981 982
			btrfs_free_reserved_data_space_noquota(inode, start,
						end - start + 1);
983
			*nr_written = *nr_written +
984
			     (end - start + PAGE_SIZE) / PAGE_SIZE;
985 986
			*page_started = 1;
			goto out;
987 988
		} else if (ret < 0) {
			goto out_unlock;
989 990 991 992
		}
	}

	BUG_ON(disk_num_bytes >
993
	       btrfs_super_total_bytes(root->fs_info->super_copy));
994

995
	alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
996 997
	btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);

998
	while (disk_num_bytes > 0) {
999 1000
		unsigned long op;

1001
		cur_alloc_size = disk_num_bytes;
1002
		ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1003
					   root->sectorsize, 0, alloc_hint,
1004
					   &ins, 1, 1);
1005
		if (ret < 0)
1006
			goto out_unlock;
1007

1008
		em = alloc_extent_map();
1009 1010
		if (!em) {
			ret = -ENOMEM;
1011
			goto out_reserve;
1012
		}
1013
		em->start = start;
1014
		em->orig_start = em->start;
1015 1016
		ram_size = ins.offset;
		em->len = ins.offset;
1017 1018
		em->mod_start = em->start;
		em->mod_len = em->len;
1019

1020
		em->block_start = ins.objectid;
1021
		em->block_len = ins.offset;
1022
		em->orig_block_len = ins.offset;
Josef Bacik's avatar
Josef Bacik committed
1023
		em->ram_bytes = ram_size;
1024
		em->bdev = root->fs_info->fs_devices->latest_bdev;
1025
		set_bit(EXTENT_FLAG_PINNED, &em->flags);
1026
		em->generation = -1;
1027

1028
		while (1) {
1029
			write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
1030
			ret = add_extent_mapping(em_tree, em, 1);
1031
			write_unlock(&em_tree->lock);
1032 1033 1034 1035 1036
			if (ret != -EEXIST) {
				free_extent_map(em);
				break;
			}
			btrfs_drop_extent_cache(inode, start,
1037
						start + ram_size - 1, 0);
1038
		}
1039 1040
		if (ret)
			goto out_reserve;
1041

1042
		cur_alloc_size = ins.offset;
1043
		ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1044
					       ram_size, cur_alloc_size, 0);
1045
		if (ret)
1046
			goto out_drop_extent_cache;
1047

1048 1049 1050 1051
		if (root->root_key.objectid ==
		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
			ret = btrfs_reloc_clone_csums(inode, start,
						      cur_alloc_size);
1052
			if (ret)
1053
				goto out_drop_extent_cache;
1054 1055
		}

1056 1057
		btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);

1058
		if (disk_num_bytes < cur_alloc_size)
1059
			break;
1060

1061 1062 1063
		/* we're not doing compressed IO, don't unlock the first
		 * page (which the caller expects to stay locked), don't
		 * clear any dirty bits and don't set any writeback bits
1064 1065 1066
		 *
		 * Do set the Private2 bit so we know this page was properly
		 * setup for writepage
1067
		 */
1068 1069
		op = unlock ? PAGE_UNLOCK : 0;
		op |= PAGE_SET_PRIVATE2;
1070

1071
		extent_clear_unlock_delalloc(inode, start,
1072 1073
					     start + ram_size - 1,
					     delalloc_end, locked_page,
1074 1075
					     EXTENT_LOCKED | EXTENT_DELALLOC,
					     op);
1076
		disk_num_bytes -= cur_alloc_size;
1077 1078 1079
		num_bytes -= cur_alloc_size;
		alloc_hint = ins.objectid + ins.offset;
		start += cur_alloc_size;
1080
	}
1081
out:
1082
	return ret;
1083

1084 1085
out_drop_extent_cache:
	btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0);
1086
out_reserve:
1087
	btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);
1088
	btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
1089
out_unlock:
1090 1091
	extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
				     locked_page,
1092 1093 1094 1095
				     EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
				     EXTENT_DELALLOC | EXTENT_DEFRAG,
				     PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
				     PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK);
1096
	goto out;
1097
}
1098

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
/*
 * work queue call back to started compression on a file and pages
 */
static noinline void async_cow_start(struct btrfs_work *work)
{
	struct async_cow *async_cow;
	int num_added = 0;
	async_cow = container_of(work, struct async_cow, work);

	compress_file_range(async_cow->inode, async_cow->locked_page,
			    async_cow->start, async_cow->end, async_cow,
			    &num_added);
1111
	if (num_added == 0) {
1112
		btrfs_add_delayed_iput(async_cow->inode);
1113
		async_cow->inode = NULL;
1114
	}
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
}

/*
 * work queue call back to submit previously compressed pages
 */
static noinline void async_cow_submit(struct btrfs_work *work)
{
	struct async_cow *async_cow;
	struct btrfs_root *root;
	unsigned long nr_pages;

	async_cow = container_of(work, struct async_cow, work);

	root = async_cow->root;
1129 1130
	nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
		PAGE_SHIFT;
1131

1132 1133 1134
	/*
	 * atomic_sub_return implies a barrier for waitqueue_active
	 */
1135
	if (atomic_sub_return(nr_pages, &root->fs_info->async_delalloc_pages) <
1136
	    5 * SZ_1M &&
1137 1138 1139
	    waitqueue_active(&root->fs_info->async_submit_wait))
		wake_up(&root->fs_info->async_submit_wait);

1140
	if (async_cow->inode)
1141 1142
		submit_compressed_extents(async_cow->inode, async_cow);
}
1143

1144 1145 1146 1147
static noinline void async_cow_free(struct btrfs_work *work)
{
	struct async_cow *async_cow;
	async_cow = container_of(work, struct async_cow, work);
1148
	if (async_cow->inode)
1149
		btrfs_add_delayed_iput(async_cow->inode);
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
	kfree(async_cow);
}

static int cow_file_range_async(struct inode *inode, struct page *locked_page,
				u64 start, u64 end, int *page_started,
				unsigned long *nr_written)
{
	struct async_cow *async_cow;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	unsigned long nr_pages;
	u64 cur_end;
1161
	int limit = 10 * SZ_1M;
1162

1163 1164
	clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
			 1, 0, NULL, GFP_NOFS);
1165
	while (start < end) {
1166
		async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1167
		BUG_ON(!async_cow); /* -ENOMEM */
1168
		async_cow->inode = igrab(inode);
1169 1170 1171 1172
		async_cow->root = root;
		async_cow->locked_page = locked_page;
		async_cow->start = start;

1173
		if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1174
		    !btrfs_test_opt(root->fs_info, FORCE_COMPRESS))
1175 1176
			cur_end = end;
		else
1177
			cur_end = min(end, start + SZ_512K - 1);
1178 1179 1180 1181

		async_cow->end = cur_end;
		INIT_LIST_HEAD(&async_cow->extents);

1182 1183 1184 1185
		btrfs_init_work(&async_cow->work,
				btrfs_delalloc_helper,
				async_cow_start, async_cow_submit,
				async_cow_free);
1186

1187 1188
		nr_pages = (cur_end - start + PAGE_SIZE) >>
			PAGE_SHIFT;
1189 1190
		atomic_add(nr_pages, &root->fs_info->async_delalloc_pages);

1191 1192
		btrfs_queue_work(root->fs_info->delalloc_workers,
				 &async_cow->work);
1193 1194 1195 1196 1197 1198 1199

		if (atomic_read(&root->fs_info->async_delalloc_pages) > limit) {
			wait_event(root->fs_info->async_submit_wait,
			   (atomic_read(&root->fs_info->async_delalloc_pages) <
			    limit));
		}

1200
		while (atomic_read(&root->fs_info->async_submit_draining) &&
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
		      atomic_read(&root->fs_info->async_delalloc_pages)) {
			wait_event(root->fs_info->async_submit_wait,
			  (atomic_read(&root->fs_info->async_delalloc_pages) ==
			   0));
		}

		*nr_written += nr_pages;
		start = cur_end + 1;
	}
	*page_started = 1;
	return 0;
1212 1213
}

1214
static noinline int csum_exist_in_range(struct btrfs_root *root,
1215 1216 1217 1218 1219 1220
					u64 bytenr, u64 num_bytes)
{
	int ret;
	struct btrfs_ordered_sum *sums;
	LIST_HEAD(list);

1221
	ret = btrfs_lookup_csums_range(root->fs_info->csum_root, bytenr,
Arne Jansen's avatar
Arne Jansen committed
1222
				       bytenr + num_bytes - 1, &list, 0);
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
	if (ret == 0 && list_empty(&list))
		return 0;

	while (!list_empty(&list)) {
		sums = list_entry(list.next, struct btrfs_ordered_sum, list);
		list_del(&sums->list);
		kfree(sums);
	}
	return 1;
}

1234 1235 1236 1237 1238 1239 1240
/*
 * when nowcow writeback call back.  This checks for snapshots or COW copies
 * of the extents that exist in the file, and COWs the file as required.
 *
 * If no cow copies or snapshots exist, we write directly to the existing
 * blocks on disk
 */
1241 1242
static noinline int run_delalloc_nocow(struct inode *inode,
				       struct page *locked_page,
1243 1244
			      u64 start, u64 end, int *page_started, int force,
			      unsigned long *nr_written)
1245 1246
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
1247
	struct btrfs_trans_handle *trans;
1248 1249
	struct extent_buffer *leaf;
	struct btrfs_path *path;
Yan Zheng's avatar
Yan Zheng committed
1250
	struct btrfs_file_extent_item *fi;
1251
	struct btrfs_key found_key;
Yan Zheng's avatar
Yan Zheng committed
1252 1253 1254
	u64 cow_start;
	u64 cur_offset;
	u64 extent_end;
1255
	u64 extent_offset;
Yan Zheng's avatar
Yan Zheng committed
1256 1257
	u64 disk_bytenr;
	u64 num_bytes;
1258
	u64 disk_num_bytes;
Josef Bacik's avatar
Josef Bacik committed
1259
	u64 ram_bytes;
Yan Zheng's avatar
Yan Zheng committed
1260
	int extent_type;
1261
	int ret, err;
Yan Zheng's avatar
Yan Zheng committed
1262
	int type;
Yan Zheng's avatar
Yan Zheng committed
1263 1264
	int nocow;
	int check_prev = 1;
1265
	bool nolock;
1266
	u64 ino = btrfs_ino(inode);
1267 1268

	path = btrfs_alloc_path();
1269
	if (!path) {
1270 1271
		extent_clear_unlock_delalloc(inode, start, end, end,
					     locked_page,
1272
					     EXTENT_LOCKED | EXTENT_DELALLOC |
1273 1274
					     EXTENT_DO_ACCOUNTING |
					     EXTENT_DEFRAG, PAGE_UNLOCK |
1275 1276 1277
					     PAGE_CLEAR_DIRTY |
					     PAGE_SET_WRITEBACK |
					     PAGE_END_WRITEBACK);
1278
		return -ENOMEM;
1279
	}
1280

1281
	nolock = btrfs_is_free_space_inode(inode);
1282 1283

	if (nolock)
1284
		trans = btrfs_join_transaction_nolock(root);
1285
	else
1286
		trans = btrfs_join_transaction(root);
Chris Mason's avatar
Chris Mason committed
1287

1288
	if (IS_ERR(trans)) {
1289 1290
		extent_clear_unlock_delalloc(inode, start, end, end,
					     locked_page,
1291
					     EXTENT_LOCKED | EXTENT_DELALLOC |
1292 1293
					     EXTENT_DO_ACCOUNTING |
					     EXTENT_DEFRAG, PAGE_UNLOCK |
1294 1295 1296
					     PAGE_CLEAR_DIRTY |
					     PAGE_SET_WRITEBACK |
					     PAGE_END_WRITEBACK);
1297 1298 1299 1300
		btrfs_free_path(path);
		return PTR_ERR(trans);
	}

1301
	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
1302

Yan Zheng's avatar
Yan Zheng committed
1303 1304 1305
	cow_start = (u64)-1;
	cur_offset = start;
	while (1) {
1306
		ret = btrfs_lookup_file_extent(trans, root, path, ino,
Yan Zheng's avatar
Yan Zheng committed
1307
					       cur_offset, 0);
1308
		if (ret < 0)
1309
			goto error;
Yan Zheng's avatar
Yan Zheng committed
1310 1311 1312 1313
		if (ret > 0 && path->slots[0] > 0 && check_prev) {
			leaf = path->nodes[0];
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0] - 1);
1314
			if (found_key.objectid == ino &&
Yan Zheng's avatar
Yan Zheng committed
1315 1316 1317 1318 1319 1320 1321 1322
			    found_key.type == BTRFS_EXTENT_DATA_KEY)
				path->slots[0]--;
		}
		check_prev = 0;
next_slot:
		leaf = path->nodes[0];
		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
1323
			if (ret < 0)
1324
				goto error;
Yan Zheng's avatar
Yan Zheng committed
1325 1326 1327 1328
			if (ret > 0)
				break;
			leaf = path->nodes[0];
		}
1329

Yan Zheng's avatar
Yan Zheng committed
1330 1331
		nocow = 0;
		disk_bytenr = 0;
1332
		num_bytes = 0;
Yan Zheng's avatar
Yan Zheng committed
1333 1334
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);

1335 1336 1337 1338 1339 1340 1341 1342
		if (found_key.objectid > ino)
			break;
		if (WARN_ON_ONCE(found_key.objectid < ino) ||
		    found_key.type < BTRFS_EXTENT_DATA_KEY) {
			path->slots[0]++;
			goto next_slot;
		}
		if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
Yan Zheng's avatar
Yan Zheng committed
1343 1344 1345 1346 1347
		    found_key.offset > end)
			break;

		if (found_key.offset > cur_offset) {
			extent_end = found_key.offset;
1348
			extent_type = 0;
Yan Zheng's avatar
Yan Zheng committed
1349 1350 1351 1352 1353 1354 1355
			goto out_check;
		}

		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
		extent_type = btrfs_file_extent_type(leaf, fi);

Josef Bacik's avatar
Josef Bacik committed
1356
		ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
Yan Zheng's avatar
Yan Zheng committed
1357 1358
		if (extent_type == BTRFS_FILE_EXTENT_REG ||
		    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
Yan Zheng's avatar
Yan Zheng committed
1359
			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1360
			extent_offset = btrfs_file_extent_offset(leaf, fi);
Yan Zheng's avatar
Yan Zheng committed
1361 1362
			extent_end = found_key.offset +
				btrfs_file_extent_num_bytes(leaf, fi);
1363 1364
			disk_num_bytes =
				btrfs_file_extent_disk_num_bytes(leaf, fi);
Yan Zheng's avatar
Yan Zheng committed
1365 1366 1367 1368
			if (extent_end <= start) {
				path->slots[0]++;
				goto next_slot;
			}
1369 1370
			if (disk_bytenr == 0)
				goto out_check;
Yan Zheng's avatar
Yan Zheng committed
1371 1372 1373 1374
			if (btrfs_file_extent_compression(leaf, fi) ||
			    btrfs_file_extent_encryption(leaf, fi) ||
			    btrfs_file_extent_other_encoding(leaf, fi))
				goto out_check;
Yan Zheng's avatar
Yan Zheng committed
1375 1376
			if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
				goto out_check;
1377
			if (btrfs_extent_readonly(root, disk_bytenr))
Yan Zheng's avatar
Yan Zheng committed
1378
				goto out_check;
1379
			if (btrfs_cross_ref_exist(trans, root, ino,
1380 1381
						  found_key.offset -
						  extent_offset, disk_bytenr))
1382
				goto out_check;
1383
			disk_bytenr += extent_offset;
1384 1385
			disk_bytenr += cur_offset - found_key.offset;
			num_bytes = min(end + 1, extent_end) - cur_offset;
1386 1387 1388 1389 1390
			/*
			 * if there are pending snapshots for this root,
			 * we fall into common COW way.
			 */
			if (!nolock) {
1391
				err = btrfs_start_write_no_snapshoting(root);
1392 1393 1394
				if (!err)
					goto out_check;
			}
1395 1396 1397 1398 1399 1400 1401
			/*
			 * force cow if csum exists in the range.
			 * this ensure that csum for a given extent are
			 * either valid or do not exist.
			 */
			if (csum_exist_in_range(root, disk_bytenr, num_bytes))
				goto out_check;
1402 1403 1404
			if (!btrfs_inc_nocow_writers(root->fs_info,
						     disk_bytenr))
				goto out_check;
Yan Zheng's avatar
Yan Zheng committed
1405 1406 1407
			nocow = 1;
		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
			extent_end = found_key.offset +
1408 1409
				btrfs_file_extent_inline_len(leaf,
						     path->slots[0], fi);
Yan Zheng's avatar
Yan Zheng committed
1410 1411 1412 1413 1414 1415 1416
			extent_end = ALIGN(extent_end, root->sectorsize);
		} else {
			BUG_ON(1);
		}
out_check:
		if (extent_end <= start) {
			path->slots[0]++;
1417
			if (!nolock && nocow)
1418
				btrfs_end_write_no_snapshoting(root);
1419 1420 1421
			if (nocow)
				btrfs_dec_nocow_writers(root->fs_info,
							disk_bytenr);
Yan Zheng's avatar
Yan Zheng committed
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
			goto next_slot;
		}
		if (!nocow) {
			if (cow_start == (u64)-1)
				cow_start = cur_offset;
			cur_offset = extent_end;
			if (cur_offset > end)
				break;
			path->slots[0]++;
			goto next_slot;
1432 1433
		}

1434
		btrfs_release_path(path);
Yan Zheng's avatar
Yan Zheng committed
1435
		if (cow_start != (u64)-1) {
1436 1437
			ret = cow_file_range(inode, locked_page,
					     cow_start, found_key.offset - 1,
1438 1439
					     end, page_started, nr_written, 1,
					     NULL);
1440 1441
			if (ret) {
				if (!nolock && nocow)
1442
					btrfs_end_write_no_snapshoting(root);
1443 1444 1445
				if (nocow)
					btrfs_dec_nocow_writers(root->fs_info,
								disk_bytenr);
1446
				goto error;
1447
			}
Yan Zheng's avatar
Yan Zheng committed
1448
			cow_start = (u64)-1;
1449
		}
Yan Zheng's avatar
Yan Zheng committed
1450

Yan Zheng's avatar
Yan Zheng committed
1451 1452 1453 1454
		if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
			struct extent_map *em;
			struct extent_map_tree *em_tree;
			em_tree = &BTRFS_I(inode)->extent_tree;
1455
			em = alloc_extent_map();
1456
			BUG_ON(!em); /* -ENOMEM */
Yan Zheng's avatar
Yan Zheng committed
1457
			em->start = cur_offset;
1458
			em->orig_start = found_key.offset - extent_offset;
Yan Zheng's avatar
Yan Zheng committed
1459 1460 1461
			em->len = num_bytes;
			em->block_len = num_bytes;
			em->block_start = disk_bytenr;
1462
			em->orig_block_len = disk_num_bytes;
Josef Bacik's avatar
Josef Bacik committed
1463
			em->ram_bytes = ram_bytes;
Yan Zheng's avatar
Yan Zheng committed
1464
			em->bdev = root->fs_info->fs_devices->latest_bdev;
1465 1466
			em->mod_start = em->start;
			em->mod_len = em->len;
Yan Zheng's avatar
Yan Zheng committed
1467
			set_bit(EXTENT_FLAG_PINNED, &em->flags);
1468
			set_bit(EXTENT_FLAG_FILLING, &em->flags);
1469
			em->generation = -1;
Yan Zheng's avatar
Yan Zheng committed
1470
			while (1) {
1471
				write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
1472
				ret = add_extent_mapping(em_tree, em, 1);
1473
				write_unlock(&em_tree->lock);
Yan Zheng's avatar
Yan Zheng committed
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484
				if (ret != -EEXIST) {
					free_extent_map(em);
					break;
				}
				btrfs_drop_extent_cache(inode, em->start,
						em->start + em->len - 1, 0);
			}
			type = BTRFS_ORDERED_PREALLOC;
		} else {
			type = BTRFS_ORDERED_NOCOW;
		}
Yan Zheng's avatar
Yan Zheng committed
1485 1486

		ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
Yan Zheng's avatar
Yan Zheng committed
1487
					       num_bytes, num_bytes, type);
1488 1489
		if (nocow)
			btrfs_dec_nocow_writers(root->fs_info, disk_bytenr);
1490
		BUG_ON(ret); /* -ENOMEM */
1491

1492 1493 1494 1495
		if (root->root_key.objectid ==
		    BTRFS_DATA_RELOC_TREE_OBJECTID) {
			ret = btrfs_reloc_clone_csums(inode, cur_offset,
						      num_bytes);
1496 1497
			if (ret) {
				if (!nolock && nocow)
1498
					btrfs_end_write_no_snapshoting(root);
1499
				goto error;
1500
			}
1501 1502
		}

1503
		extent_clear_unlock_delalloc(inode, cur_offset,
1504
					     cur_offset + num_bytes - 1, end,
1505
					     locked_page, EXTENT_LOCKED |
1506 1507 1508 1509
					     EXTENT_DELALLOC |
					     EXTENT_CLEAR_DATA_RESV,
					     PAGE_UNLOCK | PAGE_SET_PRIVATE2);

1510
		if (!nolock && nocow)
1511
			btrfs_end_write_no_snapshoting(root);
Yan Zheng's avatar
Yan Zheng committed
1512 1513 1514
		cur_offset = extent_end;
		if (cur_offset > end)
			break;
1515
	}
1516
	btrfs_release_path(path);
Yan Zheng's avatar
Yan Zheng committed
1517

1518
	if (cur_offset <= end && cow_start == (u64)-1) {
Yan Zheng's avatar
Yan Zheng committed
1519
		cow_start = cur_offset;
1520 1521 1522
		cur_offset = end;
	}

Yan Zheng's avatar
Yan Zheng committed
1523
	if (cow_start != (u64)-1) {
1524 1525
		ret = cow_file_range(inode, locked_page, cow_start, end, end,
				     page_started, nr_written, 1, NULL);
1526
		if (ret)
1527
			goto error;
Yan Zheng's avatar
Yan Zheng committed
1528 1529
	}

1530
error:
1531
	err = btrfs_end_transaction(trans, root);
1532 1533 1534
	if (!ret)
		ret = err;

1535
	if (ret && cur_offset < end)
1536
		extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1537
					     locked_page, EXTENT_LOCKED |
1538 1539 1540
					     EXTENT_DELALLOC | EXTENT_DEFRAG |
					     EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
					     PAGE_CLEAR_DIRTY |
1541 1542
					     PAGE_SET_WRITEBACK |
					     PAGE_END_WRITEBACK);
1543
	btrfs_free_path(path);
1544
	return ret;
1545 1546
}

1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
{

	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
	    !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
		return 0;

	/*
	 * @defrag_bytes is a hint value, no spinlock held here,
	 * if is not zero, it means the file is defragging.
	 * Force cow if given extent needs to be defragged.
	 */
	if (BTRFS_I(inode)->defrag_bytes &&
	    test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
			   EXTENT_DEFRAG, 0, NULL))
		return 1;

	return 0;
}

1567 1568 1569
/*
 * extent_io.c call back to do delayed allocation processing
 */
1570
static int run_delalloc_range(struct inode *inode, struct page *locked_page,
1571 1572
			      u64 start, u64 end, int *page_started,
			      unsigned long *nr_written)
1573 1574
{
	int ret;
1575
	int force_cow = need_force_cow(inode, start, end);
1576

1577
	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1578
		ret = run_delalloc_nocow(inode, locked_page, start, end,
1579
					 page_started, 1, nr_written);
1580
	} else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
Yan Zheng's avatar
Yan Zheng committed
1581
		ret = run_delalloc_nocow(inode, locked_page, start, end,
1582
					 page_started, 0, nr_written);
1583
	} else if (!inode_need_compress(inode)) {
1584 1585
		ret = cow_file_range(inode, locked_page, start, end, end,
				      page_started, nr_written, 1, NULL);
1586 1587 1588
	} else {
		set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
			&BTRFS_I(inode)->runtime_flags);
1589
		ret = cow_file_range_async(inode, locked_page, start, end,
1590
					   page_started, nr_written);
1591
	}
1592 1593 1594
	return ret;
}

1595 1596
static void btrfs_split_extent_hook(struct inode *inode,
				    struct extent_state *orig, u64 split)
Josef Bacik's avatar
Josef Bacik committed
1597
{
1598 1599
	u64 size;

1600
	/* not delalloc, ignore it */
Josef Bacik's avatar
Josef Bacik committed
1601
	if (!(orig->state & EXTENT_DELALLOC))
1602
		return;
Josef Bacik's avatar
Josef Bacik committed
1603

1604 1605 1606 1607 1608 1609
	size = orig->end - orig->start + 1;
	if (size > BTRFS_MAX_EXTENT_SIZE) {
		u64 num_extents;
		u64 new_size;

		/*
1610 1611
		 * See the explanation in btrfs_merge_extent_hook, the same
		 * applies here, just in reverse.
1612 1613
		 */
		new_size = orig->end - split + 1;
1614
		num_extents = div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1615
					BTRFS_MAX_EXTENT_SIZE);
1616 1617 1618 1619 1620
		new_size = split - orig->start;
		num_extents += div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
					BTRFS_MAX_EXTENT_SIZE);
		if (div64_u64(size + BTRFS_MAX_EXTENT_SIZE - 1,
			      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1621 1622 1623
			return;
	}

1624 1625 1626
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;
	spin_unlock(&BTRFS_I(inode)->lock);
Josef Bacik's avatar
Josef Bacik committed
1627 1628 1629 1630 1631 1632 1633 1634
}

/*
 * extent_io.c merge_extent_hook, used to track merged delayed allocation
 * extents so we can keep track of new extents that are just merged onto old
 * extents, such as when we are doing sequential writes, so we can properly
 * account for the metadata space we'll need.
 */
1635 1636 1637
static void btrfs_merge_extent_hook(struct inode *inode,
				    struct extent_state *new,
				    struct extent_state *other)
Josef Bacik's avatar
Josef Bacik committed
1638
{
1639 1640 1641
	u64 new_size, old_size;
	u64 num_extents;

Josef Bacik's avatar
Josef Bacik committed
1642 1643
	/* not delalloc, ignore it */
	if (!(other->state & EXTENT_DELALLOC))
1644
		return;
Josef Bacik's avatar
Josef Bacik committed
1645

1646 1647 1648 1649
	if (new->start > other->start)
		new_size = new->end - other->start + 1;
	else
		new_size = other->end - new->start + 1;
1650 1651 1652 1653 1654 1655 1656 1657 1658 1659

	/* we're not bigger than the max, unreserve the space and go */
	if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
		spin_lock(&BTRFS_I(inode)->lock);
		BTRFS_I(inode)->outstanding_extents--;
		spin_unlock(&BTRFS_I(inode)->lock);
		return;
	}

	/*
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
	 * We have to add up either side to figure out how many extents were
	 * accounted for before we merged into one big extent.  If the number of
	 * extents we accounted for is <= the amount we need for the new range
	 * then we can return, otherwise drop.  Think of it like this
	 *
	 * [ 4k][MAX_SIZE]
	 *
	 * So we've grown the extent by a MAX_SIZE extent, this would mean we
	 * need 2 outstanding extents, on one side we have 1 and the other side
	 * we have 1 so they are == and we can return.  But in this case
	 *
	 * [MAX_SIZE+4k][MAX_SIZE+4k]
	 *
	 * Each range on their own accounts for 2 extents, but merged together
	 * they are only 3 extents worth of accounting, so we need to drop in
	 * this case.
1676
	 */
1677
	old_size = other->end - other->start + 1;
1678 1679
	num_extents = div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
				BTRFS_MAX_EXTENT_SIZE);
1680 1681 1682 1683
	old_size = new->end - new->start + 1;
	num_extents += div64_u64(old_size + BTRFS_MAX_EXTENT_SIZE - 1,
				 BTRFS_MAX_EXTENT_SIZE);

1684
	if (div64_u64(new_size + BTRFS_MAX_EXTENT_SIZE - 1,
1685
		      BTRFS_MAX_EXTENT_SIZE) >= num_extents)
1686 1687
		return;

1688 1689 1690
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents--;
	spin_unlock(&BTRFS_I(inode)->lock);
Josef Bacik's avatar
Josef Bacik committed
1691 1692
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
				      struct inode *inode)
{
	spin_lock(&root->delalloc_lock);
	if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
		list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
			      &root->delalloc_inodes);
		set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
			&BTRFS_I(inode)->runtime_flags);
		root->nr_delalloc_inodes++;
		if (root->nr_delalloc_inodes == 1) {
			spin_lock(&root->fs_info->delalloc_root_lock);
			BUG_ON(!list_empty(&root->delalloc_root));
			list_add_tail(&root->delalloc_root,
				      &root->fs_info->delalloc_roots);
			spin_unlock(&root->fs_info->delalloc_root_lock);
		}
	}
	spin_unlock(&root->delalloc_lock);
}

static void btrfs_del_delalloc_inode(struct btrfs_root *root,
				     struct inode *inode)
{
	spin_lock(&root->delalloc_lock);
	if (!list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
		list_del_init(&BTRFS_I(inode)->delalloc_inodes);
		clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
			  &BTRFS_I(inode)->runtime_flags);
		root->nr_delalloc_inodes--;
		if (!root->nr_delalloc_inodes) {
			spin_lock(&root->fs_info->delalloc_root_lock);
			BUG_ON(list_empty(&root->delalloc_root));
			list_del_init(&root->delalloc_root);
			spin_unlock(&root->fs_info->delalloc_root_lock);
		}
	}
	spin_unlock(&root->delalloc_lock);
}

1733 1734 1735 1736 1737
/*
 * extent_io.c set_bit_hook, used to track delayed allocation
 * bytes in this file, and to maintain the list of inodes that
 * have pending delalloc work to be done.
 */
1738
static void btrfs_set_bit_hook(struct inode *inode,
1739
			       struct extent_state *state, unsigned *bits)
1740
{
Josef Bacik's avatar
Josef Bacik committed
1741

1742 1743
	if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
		WARN_ON(1);
1744 1745
	/*
	 * set_bit and clear bit hooks normally require _irqsave/restore
1746
	 * but in this case, we are only testing for the DELALLOC
1747 1748
	 * bit, which is only set or cleared with irqs on
	 */
1749
	if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1750
		struct btrfs_root *root = BTRFS_I(inode)->root;
1751
		u64 len = state->end + 1 - state->start;
1752
		bool do_list = !btrfs_is_free_space_inode(inode);
Josef Bacik's avatar
Josef Bacik committed
1753

1754
		if (*bits & EXTENT_FIRST_DELALLOC) {
1755
			*bits &= ~EXTENT_FIRST_DELALLOC;
1756 1757 1758 1759 1760
		} else {
			spin_lock(&BTRFS_I(inode)->lock);
			BTRFS_I(inode)->outstanding_extents++;
			spin_unlock(&BTRFS_I(inode)->lock);
		}
1761

1762
		/* For sanity tests */
1763
		if (btrfs_is_testing(root->fs_info))
1764 1765
			return;

1766 1767
		__percpu_counter_add(&root->fs_info->delalloc_bytes, len,
				     root->fs_info->delalloc_batch);
1768
		spin_lock(&BTRFS_I(inode)->lock);
1769
		BTRFS_I(inode)->delalloc_bytes += len;
1770 1771
		if (*bits & EXTENT_DEFRAG)
			BTRFS_I(inode)->defrag_bytes += len;
1772
		if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1773 1774
					 &BTRFS_I(inode)->runtime_flags))
			btrfs_add_delalloc_inodes(root, inode);
1775
		spin_unlock(&BTRFS_I(inode)->lock);
1776 1777 1778
	}
}

1779 1780 1781
/*
 * extent_io.c clear_bit_hook, see set_bit_hook for why
 */
1782
static void btrfs_clear_bit_hook(struct inode *inode,
1783
				 struct extent_state *state,
1784
				 unsigned *bits)
1785
{
1786
	u64 len = state->end + 1 - state->start;
1787 1788
	u64 num_extents = div64_u64(len + BTRFS_MAX_EXTENT_SIZE -1,
				    BTRFS_MAX_EXTENT_SIZE);
1789 1790 1791 1792 1793 1794

	spin_lock(&BTRFS_I(inode)->lock);
	if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
		BTRFS_I(inode)->defrag_bytes -= len;
	spin_unlock(&BTRFS_I(inode)->lock);

1795 1796
	/*
	 * set_bit and clear bit hooks normally require _irqsave/restore
1797
	 * but in this case, we are only testing for the DELALLOC
1798 1799
	 * bit, which is only set or cleared with irqs on
	 */
1800
	if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1801
		struct btrfs_root *root = BTRFS_I(inode)->root;
1802
		bool do_list = !btrfs_is_free_space_inode(inode);
1803

1804
		if (*bits & EXTENT_FIRST_DELALLOC) {
1805
			*bits &= ~EXTENT_FIRST_DELALLOC;
1806 1807
		} else if (!(*bits & EXTENT_DO_ACCOUNTING)) {
			spin_lock(&BTRFS_I(inode)->lock);
1808
			BTRFS_I(inode)->outstanding_extents -= num_extents;
1809 1810
			spin_unlock(&BTRFS_I(inode)->lock);
		}
1811

1812 1813 1814 1815 1816 1817 1818
		/*
		 * We don't reserve metadata space for space cache inodes so we
		 * don't need to call dellalloc_release_metadata if there is an
		 * error.
		 */
		if (*bits & EXTENT_DO_ACCOUNTING &&
		    root != root->fs_info->tree_root)
1819 1820
			btrfs_delalloc_release_metadata(inode, len);

1821
		/* For sanity tests. */
1822
		if (btrfs_is_testing(root->fs_info))
1823 1824
			return;

1825
		if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
1826 1827 1828
		    && do_list && !(state->state & EXTENT_NORESERVE)
		    && (*bits & (EXTENT_DO_ACCOUNTING |
		    EXTENT_CLEAR_DATA_RESV)))
1829 1830
			btrfs_free_reserved_data_space_noquota(inode,
					state->start, len);
Josef Bacik's avatar
Josef Bacik committed
1831

1832 1833
		__percpu_counter_add(&root->fs_info->delalloc_bytes, -len,
				     root->fs_info->delalloc_batch);
1834
		spin_lock(&BTRFS_I(inode)->lock);
1835
		BTRFS_I(inode)->delalloc_bytes -= len;
1836
		if (do_list && BTRFS_I(inode)->delalloc_bytes == 0 &&
1837
		    test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1838 1839
			     &BTRFS_I(inode)->runtime_flags))
			btrfs_del_delalloc_inode(root, inode);
1840
		spin_unlock(&BTRFS_I(inode)->lock);
1841 1842 1843
	}
}

1844 1845 1846
/*
 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
 * we don't create bios that span stripes or chunks
1847 1848 1849 1850
 *
 * return 1 if page cannot be merged to bio
 * return 0 if page can be merged to bio
 * return error otherwise
1851
 */
1852
int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1853 1854
			 size_t size, struct bio *bio,
			 unsigned long bio_flags)
1855 1856
{
	struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
1857
	u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1858 1859 1860 1861
	u64 length = 0;
	u64 map_length;
	int ret;

1862 1863 1864
	if (bio_flags & EXTENT_BIO_COMPRESSED)
		return 0;

1865
	length = bio->bi_iter.bi_size;
1866
	map_length = length;
1867
	ret = btrfs_map_block(root->fs_info, bio_op(bio), logical,
1868
			      &map_length, NULL, 0);
1869 1870
	if (ret < 0)
		return ret;
1871
	if (map_length < length + size)
1872
		return 1;
1873
	return 0;
1874 1875
}

1876 1877 1878 1879 1880 1881 1882 1883
/*
 * in order to insert checksums into the metadata in large chunks,
 * we wait until bio submission time.   All the pages in the bio are
 * checksummed and sums are attached onto the ordered extent record.
 *
 * At IO completion time the cums attached on the ordered extent record
 * are inserted into the btree
 */
1884 1885
static int __btrfs_submit_bio_start(struct inode *inode, struct bio *bio,
				    int mirror_num, unsigned long bio_flags,
1886
				    u64 bio_offset)
1887 1888 1889
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	int ret = 0;
1890

1891
	ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
1892
	BUG_ON(ret); /* -ENOMEM */
1893 1894
	return 0;
}
1895

1896 1897 1898 1899 1900 1901 1902 1903
/*
 * in order to insert checksums into the metadata in large chunks,
 * we wait until bio submission time.   All the pages in the bio are
 * checksummed and sums are attached onto the ordered extent record.
 *
 * At IO completion time the cums attached on the ordered extent record
 * are inserted into the btree
 */
1904
static int __btrfs_submit_bio_done(struct inode *inode, struct bio *bio,
1905 1906
			  int mirror_num, unsigned long bio_flags,
			  u64 bio_offset)
1907 1908
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
1909 1910
	int ret;

1911
	ret = btrfs_map_bio(root, bio, mirror_num, 1);
1912 1913 1914 1915
	if (ret) {
		bio->bi_error = ret;
		bio_endio(bio);
	}
1916
	return ret;
1917 1918
}

1919
/*
1920 1921
 * extent_io.c submission hook. This does the right thing for csum calculation
 * on write, or reading the csums from the tree before a read
1922
 */
1923
static int btrfs_submit_bio_hook(struct inode *inode, struct bio *bio,
1924 1925
			  int mirror_num, unsigned long bio_flags,
			  u64 bio_offset)
1926 1927
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
1928
	enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1929
	int ret = 0;
1930
	int skip_sum;
1931
	int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1932

1933
	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1934

1935
	if (btrfs_is_free_space_inode(inode))
1936
		metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1937

1938
	if (bio_op(bio) != REQ_OP_WRITE) {
1939 1940
		ret = btrfs_bio_wq_end_io(root->fs_info, bio, metadata);
		if (ret)
1941
			goto out;
1942

1943
		if (bio_flags & EXTENT_BIO_COMPRESSED) {
1944 1945 1946 1947
			ret = btrfs_submit_compressed_read(inode, bio,
							   mirror_num,
							   bio_flags);
			goto out;
1948 1949 1950
		} else if (!skip_sum) {
			ret = btrfs_lookup_bio_sums(root, inode, bio, NULL);
			if (ret)
1951
				goto out;
1952
		}
1953
		goto mapit;
1954
	} else if (async && !skip_sum) {
1955 1956 1957
		/* csum items have already been cloned */
		if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
			goto mapit;
1958
		/* we're doing a write, do the async checksumming */
1959
		ret = btrfs_wq_submit_bio(BTRFS_I(inode)->root->fs_info,
1960
				   inode, bio, mirror_num,
1961 1962
				   bio_flags, bio_offset,
				   __btrfs_submit_bio_start,
1963
				   __btrfs_submit_bio_done);
1964
		goto out;
1965 1966 1967 1968
	} else if (!skip_sum) {
		ret = btrfs_csum_one_bio(root, inode, bio, 0, 0);
		if (ret)
			goto out;
1969 1970
	}

1971
mapit:
1972
	ret = btrfs_map_bio(root, bio, mirror_num, 0);
1973 1974

out:
1975 1976 1977 1978
	if (ret < 0) {
		bio->bi_error = ret;
		bio_endio(bio);
	}
1979
	return ret;
1980
}
Chris Mason's avatar
Chris Mason committed
1981

1982 1983 1984 1985
/*
 * given a list of ordered sums record them in the inode.  This happens
 * at IO completion time based on sums calculated at bio submission time.
 */
1986
static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
1987 1988 1989 1990 1991
			     struct inode *inode, u64 file_offset,
			     struct list_head *list)
{
	struct btrfs_ordered_sum *sum;

1992
	list_for_each_entry(sum, list, list) {
1993
		trans->adding_csums = 1;
1994 1995
		btrfs_csum_file_blocks(trans,
		       BTRFS_I(inode)->root->fs_info->csum_root, sum);
1996
		trans->adding_csums = 0;
1997 1998 1999 2000
	}
	return 0;
}

2001
int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2002
			      struct extent_state **cached_state, int dedupe)
2003
{
2004
	WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2005
	return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2006
				   cached_state);
2007 2008
}

2009
/* see btrfs_writepage_start_hook for details on why this is required */
2010 2011 2012 2013 2014
struct btrfs_writepage_fixup {
	struct page *page;
	struct btrfs_work work;
};

2015
static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2016 2017 2018
{
	struct btrfs_writepage_fixup *fixup;
	struct btrfs_ordered_extent *ordered;
2019
	struct extent_state *cached_state = NULL;
2020 2021 2022 2023
	struct page *page;
	struct inode *inode;
	u64 page_start;
	u64 page_end;
2024
	int ret;
2025 2026 2027

	fixup = container_of(work, struct btrfs_writepage_fixup, work);
	page = fixup->page;
Chris Mason's avatar
Chris Mason committed
2028
again:
2029 2030 2031 2032 2033 2034 2035 2036
	lock_page(page);
	if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
		ClearPageChecked(page);
		goto out_page;
	}

	inode = page->mapping->host;
	page_start = page_offset(page);
2037
	page_end = page_offset(page) + PAGE_SIZE - 1;
2038

2039
	lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2040
			 &cached_state);
Chris Mason's avatar
Chris Mason committed
2041 2042

	/* already ordered? We're done */
2043
	if (PagePrivate2(page))
2044
		goto out;
Chris Mason's avatar
Chris Mason committed
2045

2046
	ordered = btrfs_lookup_ordered_range(inode, page_start,
2047
					PAGE_SIZE);
Chris Mason's avatar
Chris Mason committed
2048
	if (ordered) {
2049 2050
		unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
				     page_end, &cached_state, GFP_NOFS);
Chris Mason's avatar
Chris Mason committed
2051 2052
		unlock_page(page);
		btrfs_start_ordered_extent(inode, ordered, 1);
2053
		btrfs_put_ordered_extent(ordered);
Chris Mason's avatar
Chris Mason committed
2054 2055
		goto again;
	}
2056

2057
	ret = btrfs_delalloc_reserve_space(inode, page_start,
2058
					   PAGE_SIZE);
2059 2060 2061 2062 2063 2064 2065
	if (ret) {
		mapping_set_error(page->mapping, ret);
		end_extent_writepage(page, ret, page_start, page_end);
		ClearPageChecked(page);
		goto out;
	 }

2066 2067
	btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state,
				  0);
2068
	ClearPageChecked(page);
2069
	set_page_dirty(page);
2070
out:
2071 2072
	unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
			     &cached_state, GFP_NOFS);
2073 2074
out_page:
	unlock_page(page);
2075
	put_page(page);
2076
	kfree(fixup);
2077 2078 2079 2080 2081 2082 2083 2084
}

/*
 * There are a few paths in the higher layers of the kernel that directly
 * set the page dirty bit without asking the filesystem if it is a
 * good idea.  This causes problems because we want to make sure COW
 * properly happens and the data=ordered rules are followed.
 *
2085
 * In our case any range that doesn't have the ORDERED bit set
2086 2087 2088 2089
 * hasn't been properly setup for IO.  We kick off an async process
 * to fix it up.  The async helper will wait for ordered extents, set
 * the delalloc bit and make it safe to write the page.
 */
2090
static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2091 2092 2093 2094 2095
{
	struct inode *inode = page->mapping->host;
	struct btrfs_writepage_fixup *fixup;
	struct btrfs_root *root = BTRFS_I(inode)->root;

2096 2097
	/* this page is properly in the ordered list */
	if (TestClearPagePrivate2(page))
2098 2099 2100 2101 2102 2103 2104 2105
		return 0;

	if (PageChecked(page))
		return -EAGAIN;

	fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
	if (!fixup)
		return -EAGAIN;
2106

2107
	SetPageChecked(page);
2108
	get_page(page);
2109 2110
	btrfs_init_work(&fixup->work, btrfs_fixup_helper,
			btrfs_writepage_fixup_worker, NULL, NULL);
2111
	fixup->page = page;
2112
	btrfs_queue_work(root->fs_info->fixup_workers, &fixup->work);
2113
	return -EBUSY;
2114 2115
}

Yan Zheng's avatar
Yan Zheng committed
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127
static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
				       struct inode *inode, u64 file_pos,
				       u64 disk_bytenr, u64 disk_num_bytes,
				       u64 num_bytes, u64 ram_bytes,
				       u8 compression, u8 encryption,
				       u16 other_encoding, int extent_type)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_file_extent_item *fi;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_key ins;
2128
	int extent_inserted = 0;
Yan Zheng's avatar
Yan Zheng committed
2129 2130 2131
	int ret;

	path = btrfs_alloc_path();
2132 2133
	if (!path)
		return -ENOMEM;
Yan Zheng's avatar
Yan Zheng committed
2134

2135 2136 2137 2138 2139 2140 2141 2142 2143
	/*
	 * we may be replacing one extent in the tree with another.
	 * The new extent is pinned in the extent map, and we don't want
	 * to drop it from the cache until it is completely in the btree.
	 *
	 * So, tell btrfs_drop_extents to leave this extent in the cache.
	 * the caller is expected to unpin it and allow it to be merged
	 * with the others.
	 */
2144 2145 2146
	ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
				   file_pos + num_bytes, NULL, 0,
				   1, sizeof(*fi), &extent_inserted);
2147 2148
	if (ret)
		goto out;
Yan Zheng's avatar
Yan Zheng committed
2149

2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
	if (!extent_inserted) {
		ins.objectid = btrfs_ino(inode);
		ins.offset = file_pos;
		ins.type = BTRFS_EXTENT_DATA_KEY;

		path->leave_spinning = 1;
		ret = btrfs_insert_empty_item(trans, root, path, &ins,
					      sizeof(*fi));
		if (ret)
			goto out;
	}
Yan Zheng's avatar
Yan Zheng committed
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173
	leaf = path->nodes[0];
	fi = btrfs_item_ptr(leaf, path->slots[0],
			    struct btrfs_file_extent_item);
	btrfs_set_file_extent_generation(leaf, fi, trans->transid);
	btrfs_set_file_extent_type(leaf, fi, extent_type);
	btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
	btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
	btrfs_set_file_extent_offset(leaf, fi, 0);
	btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
	btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
	btrfs_set_file_extent_compression(leaf, fi, compression);
	btrfs_set_file_extent_encryption(leaf, fi, encryption);
	btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2174

Yan Zheng's avatar
Yan Zheng committed
2175
	btrfs_mark_buffer_dirty(leaf);
2176
	btrfs_release_path(path);
Yan Zheng's avatar
Yan Zheng committed
2177 2178 2179 2180 2181 2182

	inode_add_bytes(inode, num_bytes);

	ins.objectid = disk_bytenr;
	ins.offset = disk_num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2183 2184
	ret = btrfs_alloc_reserved_file_extent(trans, root,
					root->root_key.objectid,
2185 2186
					btrfs_ino(inode), file_pos,
					ram_bytes, &ins);
2187
	/*
2188 2189
	 * Release the reserved range from inode dirty range map, as it is
	 * already moved into delayed_ref_head
2190 2191
	 */
	btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2192
out:
Yan Zheng's avatar
Yan Zheng committed
2193
	btrfs_free_path(path);
2194

2195
	return ret;
Yan Zheng's avatar
Yan Zheng committed
2196 2197
}

Liu Bo's avatar
Liu Bo committed
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
/* snapshot-aware defrag */
struct sa_defrag_extent_backref {
	struct rb_node node;
	struct old_sa_defrag_extent *old;
	u64 root_id;
	u64 inum;
	u64 file_pos;
	u64 extent_offset;
	u64 num_bytes;
	u64 generation;
};

struct old_sa_defrag_extent {
	struct list_head list;
	struct new_sa_defrag_extent *new;

	u64 extent_offset;
	u64 bytenr;
	u64 offset;
	u64 len;
	int count;
};

struct new_sa_defrag_extent {
	struct rb_root root;
	struct list_head head;
	struct btrfs_path *path;
	struct inode *inode;
	u64 file_pos;
	u64 len;
	u64 bytenr;
	u64 disk_len;
	u8 compress_type;
};

static int backref_comp(struct sa_defrag_extent_backref *b1,
			struct sa_defrag_extent_backref *b2)
{
	if (b1->root_id < b2->root_id)
		return -1;
	else if (b1->root_id > b2->root_id)
		return 1;

	if (b1->inum < b2->inum)
		return -1;
	else if (b1->inum > b2->inum)
		return 1;

	if (b1->file_pos < b2->file_pos)
		return -1;
	else if (b1->file_pos > b2->file_pos)
		return 1;

	/*
	 * [------------------------------] ===> (a range of space)
	 *     |<--->|   |<---->| =============> (fs/file tree A)
	 * |<---------------------------->| ===> (fs/file tree B)
	 *
	 * A range of space can refer to two file extents in one tree while
	 * refer to only one file extent in another tree.
	 *
	 * So we may process a disk offset more than one time(two extents in A)
	 * and locate at the same extent(one extent in B), then insert two same
	 * backrefs(both refer to the extent in B).
	 */
	return 0;
}

static void backref_insert(struct rb_root *root,
			   struct sa_defrag_extent_backref *backref)
{
	struct rb_node **p = &root->rb_node;
	struct rb_node *parent = NULL;
	struct sa_defrag_extent_backref *entry;
	int ret;

	while (*p) {
		parent = *p;
		entry = rb_entry(parent, struct sa_defrag_extent_backref, node);

		ret = backref_comp(backref, entry);
		if (ret < 0)
			p = &(*p)->rb_left;
		else
			p = &(*p)->rb_right;
	}

	rb_link_node(&backref->node, parent, p);
	rb_insert_color(&backref->node, root);
}

/*
 * Note the backref might has changed, and in this case we just return 0.
 */
static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
				       void *ctx)
{
	struct btrfs_file_extent_item *extent;
	struct btrfs_fs_info *fs_info;
	struct old_sa_defrag_extent *old = ctx;
	struct new_sa_defrag_extent *new = old->new;
	struct btrfs_path *path = new->path;
	struct btrfs_key key;
	struct btrfs_root *root;
	struct sa_defrag_extent_backref *backref;
	struct extent_buffer *leaf;
	struct inode *inode = new->inode;
	int slot;
	int ret;
	u64 extent_offset;
	u64 num_bytes;

	if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
	    inum == btrfs_ino(inode))
		return 0;

	key.objectid = root_id;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = (u64)-1;

	fs_info = BTRFS_I(inode)->root->fs_info;
	root = btrfs_read_fs_root_no_name(fs_info, &key);
	if (IS_ERR(root)) {
		if (PTR_ERR(root) == -ENOENT)
			return 0;
		WARN_ON(1);
2324
		btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
Liu Bo's avatar
Liu Bo committed
2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
			 inum, offset, root_id);
		return PTR_ERR(root);
	}

	key.objectid = inum;
	key.type = BTRFS_EXTENT_DATA_KEY;
	if (offset > (u64)-1 << 32)
		key.offset = 0;
	else
		key.offset = offset;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2337
	if (WARN_ON(ret < 0))
Liu Bo's avatar
Liu Bo committed
2338
		return ret;
2339
	ret = 0;
Liu Bo's avatar
Liu Bo committed
2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373

	while (1) {
		cond_resched();

		leaf = path->nodes[0];
		slot = path->slots[0];

		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0) {
				goto out;
			} else if (ret > 0) {
				ret = 0;
				goto out;
			}
			continue;
		}

		path->slots[0]++;

		btrfs_item_key_to_cpu(leaf, &key, slot);

		if (key.objectid > inum)
			goto out;

		if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
			continue;

		extent = btrfs_item_ptr(leaf, slot,
					struct btrfs_file_extent_item);

		if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
			continue;

2374 2375 2376 2377 2378 2379
		/*
		 * 'offset' refers to the exact key.offset,
		 * NOT the 'offset' field in btrfs_extent_data_ref, ie.
		 * (key.offset - extent_offset).
		 */
		if (key.offset != offset)
Liu Bo's avatar
Liu Bo committed
2380 2381
			continue;

2382
		extent_offset = btrfs_file_extent_offset(leaf, extent);
Liu Bo's avatar
Liu Bo committed
2383
		num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2384

Liu Bo's avatar
Liu Bo committed
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399
		if (extent_offset >= old->extent_offset + old->offset +
		    old->len || extent_offset + num_bytes <=
		    old->extent_offset + old->offset)
			continue;
		break;
	}

	backref = kmalloc(sizeof(*backref), GFP_NOFS);
	if (!backref) {
		ret = -ENOENT;
		goto out;
	}

	backref->root_id = root_id;
	backref->inum = inum;
2400
	backref->file_pos = offset;
Liu Bo's avatar
Liu Bo committed
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
	backref->num_bytes = num_bytes;
	backref->extent_offset = extent_offset;
	backref->generation = btrfs_file_extent_generation(leaf, extent);
	backref->old = old;
	backref_insert(&new->root, backref);
	old->count++;
out:
	btrfs_release_path(path);
	WARN_ON(ret);
	return ret;
}

static noinline bool record_extent_backrefs(struct btrfs_path *path,
				   struct new_sa_defrag_extent *new)
{
	struct btrfs_fs_info *fs_info = BTRFS_I(new->inode)->root->fs_info;
	struct old_sa_defrag_extent *old, *tmp;
	int ret;

	new->path = path;

	list_for_each_entry_safe(old, tmp, &new->head, list) {
2423 2424
		ret = iterate_inodes_from_logical(old->bytenr +
						  old->extent_offset, fs_info,
Liu Bo's avatar
Liu Bo committed
2425 2426
						  path, record_one_backref,
						  old);
2427 2428
		if (ret < 0 && ret != -ENOENT)
			return false;
Liu Bo's avatar
Liu Bo committed
2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444

		/* no backref to be processed for this extent */
		if (!old->count) {
			list_del(&old->list);
			kfree(old);
		}
	}

	if (list_empty(&new->head))
		return false;

	return true;
}

static int relink_is_mergable(struct extent_buffer *leaf,
			      struct btrfs_file_extent_item *fi,
2445
			      struct new_sa_defrag_extent *new)
Liu Bo's avatar
Liu Bo committed
2446
{
2447
	if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
Liu Bo's avatar
Liu Bo committed
2448 2449 2450 2451 2452
		return 0;

	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
		return 0;

2453 2454 2455 2456
	if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
		return 0;

	if (btrfs_file_extent_encryption(leaf, fi) ||
Liu Bo's avatar
Liu Bo committed
2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511
	    btrfs_file_extent_other_encoding(leaf, fi))
		return 0;

	return 1;
}

/*
 * Note the backref might has changed, and in this case we just return 0.
 */
static noinline int relink_extent_backref(struct btrfs_path *path,
				 struct sa_defrag_extent_backref *prev,
				 struct sa_defrag_extent_backref *backref)
{
	struct btrfs_file_extent_item *extent;
	struct btrfs_file_extent_item *item;
	struct btrfs_ordered_extent *ordered;
	struct btrfs_trans_handle *trans;
	struct btrfs_fs_info *fs_info;
	struct btrfs_root *root;
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct old_sa_defrag_extent *old = backref->old;
	struct new_sa_defrag_extent *new = old->new;
	struct inode *src_inode = new->inode;
	struct inode *inode;
	struct extent_state *cached = NULL;
	int ret = 0;
	u64 start;
	u64 len;
	u64 lock_start;
	u64 lock_end;
	bool merge = false;
	int index;

	if (prev && prev->root_id == backref->root_id &&
	    prev->inum == backref->inum &&
	    prev->file_pos + prev->num_bytes == backref->file_pos)
		merge = true;

	/* step 1: get root */
	key.objectid = backref->root_id;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = (u64)-1;

	fs_info = BTRFS_I(src_inode)->root->fs_info;
	index = srcu_read_lock(&fs_info->subvol_srcu);

	root = btrfs_read_fs_root_no_name(fs_info, &key);
	if (IS_ERR(root)) {
		srcu_read_unlock(&fs_info->subvol_srcu, index);
		if (PTR_ERR(root) == -ENOENT)
			return 0;
		return PTR_ERR(root);
	}

2512 2513 2514 2515 2516
	if (btrfs_root_readonly(root)) {
		srcu_read_unlock(&fs_info->subvol_srcu, index);
		return 0;
	}

Liu Bo's avatar
Liu Bo committed
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
	/* step 2: get inode */
	key.objectid = backref->inum;
	key.type = BTRFS_INODE_ITEM_KEY;
	key.offset = 0;

	inode = btrfs_iget(fs_info->sb, &key, root, NULL);
	if (IS_ERR(inode)) {
		srcu_read_unlock(&fs_info->subvol_srcu, index);
		return 0;
	}

	srcu_read_unlock(&fs_info->subvol_srcu, index);

	/* step 3: relink backref */
	lock_start = backref->file_pos;
	lock_end = backref->file_pos + backref->num_bytes - 1;
	lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2534
			 &cached);
Liu Bo's avatar
Liu Bo committed
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586

	ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
	if (ordered) {
		btrfs_put_ordered_extent(ordered);
		goto out_unlock;
	}

	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		goto out_unlock;
	}

	key.objectid = backref->inum;
	key.type = BTRFS_EXTENT_DATA_KEY;
	key.offset = backref->file_pos;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0) {
		goto out_free_path;
	} else if (ret > 0) {
		ret = 0;
		goto out_free_path;
	}

	extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
				struct btrfs_file_extent_item);

	if (btrfs_file_extent_generation(path->nodes[0], extent) !=
	    backref->generation)
		goto out_free_path;

	btrfs_release_path(path);

	start = backref->file_pos;
	if (backref->extent_offset < old->extent_offset + old->offset)
		start += old->extent_offset + old->offset -
			 backref->extent_offset;

	len = min(backref->extent_offset + backref->num_bytes,
		  old->extent_offset + old->offset + old->len);
	len -= max(backref->extent_offset, old->extent_offset + old->offset);

	ret = btrfs_drop_extents(trans, root, inode, start,
				 start + len, 1);
	if (ret)
		goto out_free_path;
again:
	key.objectid = btrfs_ino(inode);
	key.type = BTRFS_EXTENT_DATA_KEY;
	key.offset = start;

2587
	path->leave_spinning = 1;
Liu Bo's avatar
Liu Bo committed
2588 2589 2590 2591 2592
	if (merge) {
		struct btrfs_file_extent_item *fi;
		u64 extent_len;
		struct btrfs_key found_key;

2593
		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
Liu Bo's avatar
Liu Bo committed
2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604
		if (ret < 0)
			goto out_free_path;

		path->slots[0]--;
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);

		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
		extent_len = btrfs_file_extent_num_bytes(leaf, fi);

2605 2606
		if (extent_len + found_key.offset == start &&
		    relink_is_mergable(leaf, fi, new)) {
Liu Bo's avatar
Liu Bo committed
2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623
			btrfs_set_file_extent_num_bytes(leaf, fi,
							extent_len + len);
			btrfs_mark_buffer_dirty(leaf);
			inode_add_bytes(inode, len);

			ret = 1;
			goto out_free_path;
		} else {
			merge = false;
			btrfs_release_path(path);
			goto again;
		}
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key,
					sizeof(*extent));
	if (ret) {
2624
		btrfs_abort_transaction(trans, ret);
Liu Bo's avatar
Liu Bo committed
2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643
		goto out_free_path;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0],
				struct btrfs_file_extent_item);
	btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
	btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
	btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
	btrfs_set_file_extent_num_bytes(leaf, item, len);
	btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
	btrfs_set_file_extent_generation(leaf, item, trans->transid);
	btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
	btrfs_set_file_extent_compression(leaf, item, new->compress_type);
	btrfs_set_file_extent_encryption(leaf, item, 0);
	btrfs_set_file_extent_other_encoding(leaf, item, 0);

	btrfs_mark_buffer_dirty(leaf);
	inode_add_bytes(inode, len);
2644
	btrfs_release_path(path);
Liu Bo's avatar
Liu Bo committed
2645 2646 2647 2648

	ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
			new->disk_len, 0,
			backref->root_id, backref->inum,
2649
			new->file_pos);	/* start - extent_offset */
Liu Bo's avatar
Liu Bo committed
2650
	if (ret) {
2651
		btrfs_abort_transaction(trans, ret);
Liu Bo's avatar
Liu Bo committed
2652 2653 2654 2655 2656 2657
		goto out_free_path;
	}

	ret = 1;
out_free_path:
	btrfs_release_path(path);
2658
	path->leave_spinning = 0;
Liu Bo's avatar
Liu Bo committed
2659 2660 2661 2662 2663 2664 2665 2666
	btrfs_end_transaction(trans, root);
out_unlock:
	unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
			     &cached, GFP_NOFS);
	iput(inode);
	return ret;
}

2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
{
	struct old_sa_defrag_extent *old, *tmp;

	if (!new)
		return;

	list_for_each_entry_safe(old, tmp, &new->head, list) {
		kfree(old);
	}
	kfree(new);
}

Liu Bo's avatar
Liu Bo committed
2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725
static void relink_file_extents(struct new_sa_defrag_extent *new)
{
	struct btrfs_path *path;
	struct sa_defrag_extent_backref *backref;
	struct sa_defrag_extent_backref *prev = NULL;
	struct inode *inode;
	struct btrfs_root *root;
	struct rb_node *node;
	int ret;

	inode = new->inode;
	root = BTRFS_I(inode)->root;

	path = btrfs_alloc_path();
	if (!path)
		return;

	if (!record_extent_backrefs(path, new)) {
		btrfs_free_path(path);
		goto out;
	}
	btrfs_release_path(path);

	while (1) {
		node = rb_first(&new->root);
		if (!node)
			break;
		rb_erase(node, &new->root);

		backref = rb_entry(node, struct sa_defrag_extent_backref, node);

		ret = relink_extent_backref(path, prev, backref);
		WARN_ON(ret < 0);

		kfree(prev);

		if (ret == 1)
			prev = backref;
		else
			prev = NULL;
		cond_resched();
	}
	kfree(prev);

	btrfs_free_path(path);
out:
2726 2727
	free_sa_defrag_extent(new);

Liu Bo's avatar
Liu Bo committed
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738
	atomic_dec(&root->fs_info->defrag_running);
	wake_up(&root->fs_info->transaction_wait);
}

static struct new_sa_defrag_extent *
record_old_file_extents(struct inode *inode,
			struct btrfs_ordered_extent *ordered)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_path *path;
	struct btrfs_key key;
2739
	struct old_sa_defrag_extent *old;
Liu Bo's avatar
Liu Bo committed
2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
	struct new_sa_defrag_extent *new;
	int ret;

	new = kmalloc(sizeof(*new), GFP_NOFS);
	if (!new)
		return NULL;

	new->inode = inode;
	new->file_pos = ordered->file_offset;
	new->len = ordered->len;
	new->bytenr = ordered->start;
	new->disk_len = ordered->disk_len;
	new->compress_type = ordered->compress_type;
	new->root = RB_ROOT;
	INIT_LIST_HEAD(&new->head);

	path = btrfs_alloc_path();
	if (!path)
		goto out_kfree;

	key.objectid = btrfs_ino(inode);
	key.type = BTRFS_EXTENT_DATA_KEY;
	key.offset = new->file_pos;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0)
		goto out_free_path;
	if (ret > 0 && path->slots[0] > 0)
		path->slots[0]--;

	/* find out all the old extents for the file range */
	while (1) {
		struct btrfs_file_extent_item *extent;
		struct extent_buffer *l;
		int slot;
		u64 num_bytes;
		u64 offset;
		u64 end;
		u64 disk_bytenr;
		u64 extent_offset;

		l = path->nodes[0];
		slot = path->slots[0];

		if (slot >= btrfs_header_nritems(l)) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
2787
				goto out_free_path;
Liu Bo's avatar
Liu Bo committed
2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815
			else if (ret > 0)
				break;
			continue;
		}

		btrfs_item_key_to_cpu(l, &key, slot);

		if (key.objectid != btrfs_ino(inode))
			break;
		if (key.type != BTRFS_EXTENT_DATA_KEY)
			break;
		if (key.offset >= new->file_pos + new->len)
			break;

		extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);

		num_bytes = btrfs_file_extent_num_bytes(l, extent);
		if (key.offset + num_bytes < new->file_pos)
			goto next;

		disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
		if (!disk_bytenr)
			goto next;

		extent_offset = btrfs_file_extent_offset(l, extent);

		old = kmalloc(sizeof(*old), GFP_NOFS);
		if (!old)
2816
			goto out_free_path;
Liu Bo's avatar
Liu Bo committed
2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840

		offset = max(new->file_pos, key.offset);
		end = min(new->file_pos + new->len, key.offset + num_bytes);

		old->bytenr = disk_bytenr;
		old->extent_offset = extent_offset;
		old->offset = offset - key.offset;
		old->len = end - offset;
		old->new = new;
		old->count = 0;
		list_add_tail(&old->list, &new->head);
next:
		path->slots[0]++;
		cond_resched();
	}

	btrfs_free_path(path);
	atomic_inc(&root->fs_info->defrag_running);

	return new;

out_free_path:
	btrfs_free_path(path);
out_kfree:
2841
	free_sa_defrag_extent(new);
Liu Bo's avatar
Liu Bo committed
2842 2843 2844
	return NULL;
}

2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859
static void btrfs_release_delalloc_bytes(struct btrfs_root *root,
					 u64 start, u64 len)
{
	struct btrfs_block_group_cache *cache;

	cache = btrfs_lookup_block_group(root->fs_info, start);
	ASSERT(cache);

	spin_lock(&cache->lock);
	cache->delalloc_bytes -= len;
	spin_unlock(&cache->lock);

	btrfs_put_block_group(cache);
}

2860 2861 2862 2863
/* as ordered data IO finishes, this gets called so we can finish
 * an ordered extent if the range of bytes in the file it covers are
 * fully written.
 */
2864
static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2865
{
2866
	struct inode *inode = ordered_extent->inode;
2867
	struct btrfs_root *root = BTRFS_I(inode)->root;
2868
	struct btrfs_trans_handle *trans = NULL;
2869
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2870
	struct extent_state *cached_state = NULL;
Liu Bo's avatar
Liu Bo committed
2871
	struct new_sa_defrag_extent *new = NULL;
2872
	int compress_type = 0;
2873 2874
	int ret = 0;
	u64 logical_len = ordered_extent->len;
2875
	bool nolock;
2876
	bool truncated = false;
2877

2878
	nolock = btrfs_is_free_space_inode(inode);
2879

2880 2881 2882 2883 2884
	if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
		ret = -EIO;
		goto out;
	}

2885 2886 2887 2888
	btrfs_free_io_failure_record(inode, ordered_extent->file_offset,
				     ordered_extent->file_offset +
				     ordered_extent->len - 1);

2889 2890 2891 2892 2893 2894 2895 2896
	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
		truncated = true;
		logical_len = ordered_extent->truncated_len;
		/* Truncated the entire extent, don't bother adding */
		if (!logical_len)
			goto out;
	}

2897
	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2898
		BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2899 2900 2901 2902 2903 2904 2905 2906

		/*
		 * For mwrite(mmap + memset to write) case, we still reserve
		 * space for NOCOW range.
		 * As NOCOW won't cause a new delayed ref, just free the space
		 */
		btrfs_qgroup_free_data(inode, ordered_extent->file_offset,
				       ordered_extent->len);
2907 2908 2909 2910 2911 2912 2913 2914 2915
		btrfs_ordered_update_i_size(inode, 0, ordered_extent);
		if (nolock)
			trans = btrfs_join_transaction_nolock(root);
		else
			trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			trans = NULL;
			goto out;
2916
		}
2917 2918 2919
		trans->block_rsv = &root->fs_info->delalloc_block_rsv;
		ret = btrfs_update_inode_fallback(trans, root, inode);
		if (ret) /* -ENOMEM or corruption */
2920
			btrfs_abort_transaction(trans, ret);
2921 2922
		goto out;
	}
2923

2924 2925
	lock_extent_bits(io_tree, ordered_extent->file_offset,
			 ordered_extent->file_offset + ordered_extent->len - 1,
2926
			 &cached_state);
2927

Liu Bo's avatar
Liu Bo committed
2928 2929 2930 2931 2932
	ret = test_range_bit(io_tree, ordered_extent->file_offset,
			ordered_extent->file_offset + ordered_extent->len - 1,
			EXTENT_DEFRAG, 1, cached_state);
	if (ret) {
		u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2933
		if (0 && last_snapshot >= BTRFS_I(inode)->generation)
Liu Bo's avatar
Liu Bo committed
2934 2935 2936 2937 2938 2939 2940 2941
			/* the inode is shared */
			new = record_old_file_extents(inode, ordered_extent);

		clear_extent_bit(io_tree, ordered_extent->file_offset,
			ordered_extent->file_offset + ordered_extent->len - 1,
			EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
	}

2942
	if (nolock)
2943
		trans = btrfs_join_transaction_nolock(root);
2944
	else
2945
		trans = btrfs_join_transaction(root);
2946 2947 2948 2949 2950
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		trans = NULL;
		goto out_unlock;
	}
Chris Mason's avatar
Chris Mason committed
2951

2952
	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
2953

2954
	if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
2955
		compress_type = ordered_extent->compress_type;
Yan Zheng's avatar
Yan Zheng committed
2956
	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
2957
		BUG_ON(compress_type);
2958
		ret = btrfs_mark_extent_written(trans, inode,
Yan Zheng's avatar
Yan Zheng committed
2959 2960
						ordered_extent->file_offset,
						ordered_extent->file_offset +
2961
						logical_len);
Yan Zheng's avatar
Yan Zheng committed
2962
	} else {
2963
		BUG_ON(root == root->fs_info->tree_root);
Yan Zheng's avatar
Yan Zheng committed
2964 2965 2966 2967
		ret = insert_reserved_file_extent(trans, inode,
						ordered_extent->file_offset,
						ordered_extent->start,
						ordered_extent->disk_len,
2968
						logical_len, logical_len,
2969
						compress_type, 0, 0,
Yan Zheng's avatar
Yan Zheng committed
2970
						BTRFS_FILE_EXTENT_REG);
2971 2972 2973 2974
		if (!ret)
			btrfs_release_delalloc_bytes(root,
						     ordered_extent->start,
						     ordered_extent->disk_len);
Yan Zheng's avatar
Yan Zheng committed
2975
	}
Josef Bacik's avatar
Josef Bacik committed
2976 2977 2978
	unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
			   ordered_extent->file_offset, ordered_extent->len,
			   trans->transid);
2979
	if (ret < 0) {
2980
		btrfs_abort_transaction(trans, ret);
2981
		goto out_unlock;
2982
	}
2983

2984 2985 2986
	add_pending_csums(trans, inode, ordered_extent->file_offset,
			  &ordered_extent->list);

2987 2988 2989
	btrfs_ordered_update_i_size(inode, 0, ordered_extent);
	ret = btrfs_update_inode_fallback(trans, root, inode);
	if (ret) { /* -ENOMEM or corruption */
2990
		btrfs_abort_transaction(trans, ret);
2991
		goto out_unlock;
2992 2993
	}
	ret = 0;
2994 2995 2996 2997
out_unlock:
	unlock_extent_cached(io_tree, ordered_extent->file_offset,
			     ordered_extent->file_offset +
			     ordered_extent->len - 1, &cached_state, GFP_NOFS);
2998
out:
2999
	if (root != root->fs_info->tree_root)
3000
		btrfs_delalloc_release_metadata(inode, ordered_extent->len);
3001 3002
	if (trans)
		btrfs_end_transaction(trans, root);
3003

3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
	if (ret || truncated) {
		u64 start, end;

		if (truncated)
			start = ordered_extent->file_offset + logical_len;
		else
			start = ordered_extent->file_offset;
		end = ordered_extent->file_offset + ordered_extent->len - 1;
		clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);

		/* Drop the cache for the part of the extent we didn't write. */
		btrfs_drop_extent_cache(inode, start, end, 0);
3016

3017 3018 3019
		/*
		 * If the ordered extent had an IOERR or something else went
		 * wrong we need to return the space for this ordered extent
3020 3021
		 * back to the allocator.  We only free the extent in the
		 * truncated case if we didn't write out the extent at all.
3022
		 */
3023 3024
		if ((ret || !logical_len) &&
		    !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3025 3026
		    !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
			btrfs_free_reserved_extent(root, ordered_extent->start,
3027
						   ordered_extent->disk_len, 1);
3028 3029 3030
	}


3031
	/*
3032 3033
	 * This needs to be done to make sure anybody waiting knows we are done
	 * updating everything for this ordered extent.
3034 3035 3036
	 */
	btrfs_remove_ordered_extent(inode, ordered_extent);

Liu Bo's avatar
Liu Bo committed
3037
	/* for snapshot-aware defrag */
3038 3039 3040 3041 3042 3043 3044 3045
	if (new) {
		if (ret) {
			free_sa_defrag_extent(new);
			atomic_dec(&root->fs_info->defrag_running);
		} else {
			relink_file_extents(new);
		}
	}
Liu Bo's avatar
Liu Bo committed
3046

3047 3048 3049 3050 3051
	/* once for us */
	btrfs_put_ordered_extent(ordered_extent);
	/* once for the tree */
	btrfs_put_ordered_extent(ordered_extent);

3052 3053 3054 3055 3056 3057 3058 3059
	return ret;
}

static void finish_ordered_fn(struct btrfs_work *work)
{
	struct btrfs_ordered_extent *ordered_extent;
	ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
	btrfs_finish_ordered_io(ordered_extent);
3060 3061
}

3062
static int btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3063 3064
				struct extent_state *state, int uptodate)
{
3065 3066 3067
	struct inode *inode = page->mapping->host;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_ordered_extent *ordered_extent = NULL;
3068 3069
	struct btrfs_workqueue *wq;
	btrfs_work_func_t func;
3070

3071 3072
	trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);

3073
	ClearPagePrivate2(page);
3074 3075 3076 3077
	if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
					    end - start + 1, uptodate))
		return 0;

3078 3079 3080 3081 3082 3083 3084
	if (btrfs_is_free_space_inode(inode)) {
		wq = root->fs_info->endio_freespace_worker;
		func = btrfs_freespace_write_helper;
	} else {
		wq = root->fs_info->endio_write_workers;
		func = btrfs_endio_write_helper;
	}
3085

3086 3087 3088
	btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
			NULL);
	btrfs_queue_work(wq, &ordered_extent->work);
3089 3090

	return 0;
3091 3092
}

3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112
static int __readpage_endio_check(struct inode *inode,
				  struct btrfs_io_bio *io_bio,
				  int icsum, struct page *page,
				  int pgoff, u64 start, size_t len)
{
	char *kaddr;
	u32 csum_expected;
	u32 csum = ~(u32)0;

	csum_expected = *(((u32 *)io_bio->csum) + icsum);

	kaddr = kmap_atomic(page);
	csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
	btrfs_csum_final(csum, (char *)&csum);
	if (csum != csum_expected)
		goto zeroit;

	kunmap_atomic(kaddr);
	return 0;
zeroit:
3113 3114
	btrfs_warn_rl(BTRFS_I(inode)->root->fs_info,
		"csum failed ino %llu off %llu csum %u expected csum %u",
3115 3116 3117 3118 3119 3120 3121 3122 3123
			   btrfs_ino(inode), start, csum, csum_expected);
	memset(kaddr + pgoff, 1, len);
	flush_dcache_page(page);
	kunmap_atomic(kaddr);
	if (csum_expected == 0)
		return 0;
	return -EIO;
}

3124 3125
/*
 * when reads are done, we need to check csums to verify the data is correct
3126 3127
 * if there's a match, we allow the bio to finish.  If not, the code in
 * extent_io.c will try to find good copies for us.
3128
 */
3129 3130 3131
static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
				      u64 phy_offset, struct page *page,
				      u64 start, u64 end, int mirror)
3132
{
Miao Xie's avatar
Miao Xie committed
3133
	size_t offset = start - page_offset(page);
3134
	struct inode *inode = page->mapping->host;
3135
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3136
	struct btrfs_root *root = BTRFS_I(inode)->root;
3137

3138 3139
	if (PageChecked(page)) {
		ClearPageChecked(page);
3140
		return 0;
3141
	}
3142 3143

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3144
		return 0;
3145 3146

	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3147
	    test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3148
		clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3149
		return 0;
3150
	}
3151

3152
	phy_offset >>= inode->i_sb->s_blocksize_bits;
3153 3154
	return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
				      start, (size_t)(end - start + 1));
3155
}
3156

Yan, Zheng's avatar
Yan, Zheng committed
3157 3158 3159
void btrfs_add_delayed_iput(struct inode *inode)
{
	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3160
	struct btrfs_inode *binode = BTRFS_I(inode);
Yan, Zheng's avatar
Yan, Zheng committed
3161 3162 3163 3164 3165

	if (atomic_add_unless(&inode->i_count, -1, 1))
		return;

	spin_lock(&fs_info->delayed_iput_lock);
3166 3167 3168 3169 3170 3171
	if (binode->delayed_iput_count == 0) {
		ASSERT(list_empty(&binode->delayed_iput));
		list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
	} else {
		binode->delayed_iput_count++;
	}
Yan, Zheng's avatar
Yan, Zheng committed
3172 3173 3174 3175 3176 3177 3178 3179
	spin_unlock(&fs_info->delayed_iput_lock);
}

void btrfs_run_delayed_iputs(struct btrfs_root *root)
{
	struct btrfs_fs_info *fs_info = root->fs_info;

	spin_lock(&fs_info->delayed_iput_lock);
3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194
	while (!list_empty(&fs_info->delayed_iputs)) {
		struct btrfs_inode *inode;

		inode = list_first_entry(&fs_info->delayed_iputs,
				struct btrfs_inode, delayed_iput);
		if (inode->delayed_iput_count) {
			inode->delayed_iput_count--;
			list_move_tail(&inode->delayed_iput,
					&fs_info->delayed_iputs);
		} else {
			list_del_init(&inode->delayed_iput);
		}
		spin_unlock(&fs_info->delayed_iput_lock);
		iput(&inode->vfs_inode);
		spin_lock(&fs_info->delayed_iput_lock);
Yan, Zheng's avatar
Yan, Zheng committed
3195
	}
3196
	spin_unlock(&fs_info->delayed_iput_lock);
Yan, Zheng's avatar
Yan, Zheng committed
3197 3198
}

3199
/*
3200
 * This is called in transaction commit time. If there are no orphan
3201 3202 3203 3204 3205 3206
 * files in the subvolume, it removes orphan item and frees block_rsv
 * structure.
 */
void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root)
{
3207
	struct btrfs_block_rsv *block_rsv;
3208 3209
	int ret;

3210
	if (atomic_read(&root->orphan_inodes) ||
3211 3212 3213
	    root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
		return;

3214
	spin_lock(&root->orphan_lock);
3215
	if (atomic_read(&root->orphan_inodes)) {
3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228
		spin_unlock(&root->orphan_lock);
		return;
	}

	if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
		spin_unlock(&root->orphan_lock);
		return;
	}

	block_rsv = root->orphan_block_rsv;
	root->orphan_block_rsv = NULL;
	spin_unlock(&root->orphan_lock);

3229
	if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3230 3231 3232
	    btrfs_root_refs(&root->root_item) > 0) {
		ret = btrfs_del_orphan_item(trans, root->fs_info->tree_root,
					    root->root_key.objectid);
3233
		if (ret)
3234
			btrfs_abort_transaction(trans, ret);
3235
		else
3236 3237
			clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
				  &root->state);
3238 3239
	}

3240 3241 3242
	if (block_rsv) {
		WARN_ON(block_rsv->size > 0);
		btrfs_free_block_rsv(root, block_rsv);
3243 3244 3245
	}
}

3246 3247 3248
/*
 * This creates an orphan entry for the given inode in case something goes
 * wrong in the middle of an unlink/truncate.
3249 3250 3251
 *
 * NOTE: caller of this function should reserve 5 units of metadata for
 *	 this function.
3252 3253 3254 3255
 */
int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
3256 3257 3258 3259
	struct btrfs_block_rsv *block_rsv = NULL;
	int reserve = 0;
	int insert = 0;
	int ret;
3260

3261
	if (!root->orphan_block_rsv) {
3262
		block_rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
3263 3264
		if (!block_rsv)
			return -ENOMEM;
3265
	}
3266

3267 3268 3269 3270 3271 3272
	spin_lock(&root->orphan_lock);
	if (!root->orphan_block_rsv) {
		root->orphan_block_rsv = block_rsv;
	} else if (block_rsv) {
		btrfs_free_block_rsv(root, block_rsv);
		block_rsv = NULL;
3273 3274
	}

3275 3276
	if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
			      &BTRFS_I(inode)->runtime_flags)) {
3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288
#if 0
		/*
		 * For proper ENOSPC handling, we should do orphan
		 * cleanup when mounting. But this introduces backward
		 * compatibility issue.
		 */
		if (!xchg(&root->orphan_item_inserted, 1))
			insert = 2;
		else
			insert = 1;
#endif
		insert = 1;
3289
		atomic_inc(&root->orphan_inodes);
3290 3291
	}

3292 3293
	if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
			      &BTRFS_I(inode)->runtime_flags))
3294 3295
		reserve = 1;
	spin_unlock(&root->orphan_lock);
3296

3297 3298 3299
	/* grab metadata reservation from transaction handle */
	if (reserve) {
		ret = btrfs_orphan_reserve_metadata(trans, inode);
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
		ASSERT(!ret);
		if (ret) {
			atomic_dec(&root->orphan_inodes);
			clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
				  &BTRFS_I(inode)->runtime_flags);
			if (insert)
				clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
					  &BTRFS_I(inode)->runtime_flags);
			return ret;
		}
3310
	}
3311

3312 3313
	/* insert an orphan item to track this unlinked/truncated file */
	if (insert >= 1) {
3314
		ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3315
		if (ret) {
3316
			atomic_dec(&root->orphan_inodes);
3317 3318 3319 3320 3321 3322
			if (reserve) {
				clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
					  &BTRFS_I(inode)->runtime_flags);
				btrfs_orphan_release_metadata(inode);
			}
			if (ret != -EEXIST) {
3323 3324
				clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
					  &BTRFS_I(inode)->runtime_flags);
3325
				btrfs_abort_transaction(trans, ret);
3326 3327
				return ret;
			}
3328 3329
		}
		ret = 0;
3330 3331 3332 3333 3334 3335
	}

	/* insert an orphan item to track subvolume contains orphan files */
	if (insert >= 2) {
		ret = btrfs_insert_orphan_item(trans, root->fs_info->tree_root,
					       root->root_key.objectid);
3336
		if (ret && ret != -EEXIST) {
3337
			btrfs_abort_transaction(trans, ret);
3338 3339
			return ret;
		}
3340 3341
	}
	return 0;
3342 3343 3344 3345 3346 3347
}

/*
 * We have done the truncate/delete so we can go ahead and remove the orphan
 * item for this particular inode.
 */
3348 3349
static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
			    struct inode *inode)
3350 3351
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
3352 3353
	int delete_item = 0;
	int release_rsv = 0;
3354 3355
	int ret = 0;

3356
	spin_lock(&root->orphan_lock);
3357 3358
	if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
			       &BTRFS_I(inode)->runtime_flags))
3359
		delete_item = 1;
3360

3361 3362
	if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
3363 3364
		release_rsv = 1;
	spin_unlock(&root->orphan_lock);
3365

3366
	if (delete_item) {
3367
		atomic_dec(&root->orphan_inodes);
3368 3369 3370
		if (trans)
			ret = btrfs_del_orphan_item(trans, root,
						    btrfs_ino(inode));
3371
	}
3372

3373 3374 3375
	if (release_rsv)
		btrfs_orphan_release_metadata(inode);

3376
	return ret;
3377 3378 3379 3380 3381 3382
}

/*
 * this cleans up any orphans that may be left on the list from the last use
 * of this root.
 */
3383
int btrfs_orphan_cleanup(struct btrfs_root *root)
3384 3385 3386 3387 3388 3389
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_key key, found_key;
	struct btrfs_trans_handle *trans;
	struct inode *inode;
3390
	u64 last_objectid = 0;
3391 3392
	int ret = 0, nr_unlink = 0, nr_truncate = 0;

3393
	if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3394
		return 0;
3395 3396

	path = btrfs_alloc_path();
3397 3398 3399 3400
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
3401
	path->reada = READA_BACK;
3402 3403

	key.objectid = BTRFS_ORPHAN_OBJECTID;
3404
	key.type = BTRFS_ORPHAN_ITEM_KEY;
3405 3406 3407 3408
	key.offset = (u64)-1;

	while (1) {
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3409 3410
		if (ret < 0)
			goto out;
3411 3412 3413

		/*
		 * if ret == 0 means we found what we were searching for, which
Lucas De Marchi's avatar
Lucas De Marchi committed
3414
		 * is weird, but possible, so only screw with path if we didn't
3415 3416 3417
		 * find the key and see if we have stuff that matches
		 */
		if (ret > 0) {
3418
			ret = 0;
3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430
			if (path->slots[0] == 0)
				break;
			path->slots[0]--;
		}

		/* pull out the item */
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);

		/* make sure the item matches what we want */
		if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
			break;
3431
		if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3432 3433 3434
			break;

		/* release the path since we're done with it */
3435
		btrfs_release_path(path);
3436 3437 3438 3439 3440 3441

		/*
		 * this is where we are basically btrfs_lookup, without the
		 * crossing root thing.  we store the inode number in the
		 * offset of the orphan item.
		 */
3442 3443

		if (found_key.offset == last_objectid) {
3444 3445
			btrfs_err(root->fs_info,
				"Error removing orphan entry, stopping orphan cleanup");
3446 3447 3448 3449 3450 3451
			ret = -EINVAL;
			goto out;
		}

		last_objectid = found_key.offset;

3452 3453 3454
		found_key.objectid = found_key.offset;
		found_key.type = BTRFS_INODE_ITEM_KEY;
		found_key.offset = 0;
3455
		inode = btrfs_iget(root->fs_info->sb, &found_key, root, NULL);
3456
		ret = PTR_ERR_OR_ZERO(inode);
3457
		if (ret && ret != -ENOENT)
3458
			goto out;
3459

3460
		if (ret == -ENOENT && root == root->fs_info->tree_root) {
3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491
			struct btrfs_root *dead_root;
			struct btrfs_fs_info *fs_info = root->fs_info;
			int is_dead_root = 0;

			/*
			 * this is an orphan in the tree root. Currently these
			 * could come from 2 sources:
			 *  a) a snapshot deletion in progress
			 *  b) a free space cache inode
			 * We need to distinguish those two, as the snapshot
			 * orphan must not get deleted.
			 * find_dead_roots already ran before us, so if this
			 * is a snapshot deletion, we should find the root
			 * in the dead_roots list
			 */
			spin_lock(&fs_info->trans_lock);
			list_for_each_entry(dead_root, &fs_info->dead_roots,
					    root_list) {
				if (dead_root->root_key.objectid ==
				    found_key.objectid) {
					is_dead_root = 1;
					break;
				}
			}
			spin_unlock(&fs_info->trans_lock);
			if (is_dead_root) {
				/* prevent this orphan from being found again */
				key.offset = found_key.objectid - 1;
				continue;
			}
		}
3492
		/*
3493 3494
		 * Inode is already gone but the orphan item is still there,
		 * kill the orphan item.
3495
		 */
3496
		if (ret == -ENOENT) {
3497
			trans = btrfs_start_transaction(root, 1);
3498 3499 3500 3501
			if (IS_ERR(trans)) {
				ret = PTR_ERR(trans);
				goto out;
			}
3502 3503
			btrfs_debug(root->fs_info, "auto deleting %Lu",
				found_key.objectid);
3504 3505
			ret = btrfs_del_orphan_item(trans, root,
						    found_key.objectid);
3506
			btrfs_end_transaction(trans, root);
3507 3508
			if (ret)
				goto out;
3509 3510 3511
			continue;
		}

3512 3513 3514 3515
		/*
		 * add this inode to the orphan list so btrfs_orphan_del does
		 * the proper thing when we hit it
		 */
3516 3517
		set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
			&BTRFS_I(inode)->runtime_flags);
3518
		atomic_inc(&root->orphan_inodes);
3519

3520 3521
		/* if we have links, this was a truncate, lets do that */
		if (inode->i_nlink) {
3522
			if (WARN_ON(!S_ISREG(inode->i_mode))) {
3523 3524 3525
				iput(inode);
				continue;
			}
3526
			nr_truncate++;
3527 3528 3529 3530

			/* 1 for the orphan item deletion. */
			trans = btrfs_start_transaction(root, 1);
			if (IS_ERR(trans)) {
3531
				iput(inode);
3532 3533 3534 3535 3536
				ret = PTR_ERR(trans);
				goto out;
			}
			ret = btrfs_orphan_add(trans, inode);
			btrfs_end_transaction(trans, root);
3537 3538
			if (ret) {
				iput(inode);
3539
				goto out;
3540
			}
3541

3542
			ret = btrfs_truncate(inode);
3543 3544
			if (ret)
				btrfs_orphan_del(NULL, inode);
3545 3546 3547 3548 3549 3550
		} else {
			nr_unlink++;
		}

		/* this will do delete_inode and everything for us */
		iput(inode);
3551 3552
		if (ret)
			goto out;
3553
	}
3554 3555 3556
	/* release the path since we're done with it */
	btrfs_release_path(path);

3557 3558 3559 3560 3561 3562
	root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;

	if (root->orphan_block_rsv)
		btrfs_block_rsv_release(root, root->orphan_block_rsv,
					(u64)-1);

3563 3564
	if (root->orphan_block_rsv ||
	    test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3565
		trans = btrfs_join_transaction(root);
3566 3567
		if (!IS_ERR(trans))
			btrfs_end_transaction(trans, root);
3568
	}
3569 3570

	if (nr_unlink)
3571
		btrfs_debug(root->fs_info, "unlinked %d orphans", nr_unlink);
3572
	if (nr_truncate)
3573
		btrfs_debug(root->fs_info, "truncated %d orphans", nr_truncate);
3574 3575 3576

out:
	if (ret)
3577
		btrfs_err(root->fs_info,
3578
			"could not do orphan cleanup %d", ret);
3579 3580
	btrfs_free_path(path);
	return ret;
3581 3582
}

3583 3584 3585 3586 3587 3588 3589
/*
 * very simple check to peek ahead in the leaf looking for xattrs.  If we
 * don't find any xattrs, we know there can't be any acls.
 *
 * slot is the slot the inode is in, objectid is the objectid of the inode
 */
static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3590 3591
					  int slot, u64 objectid,
					  int *first_xattr_slot)
3592 3593 3594
{
	u32 nritems = btrfs_header_nritems(leaf);
	struct btrfs_key found_key;
3595 3596
	static u64 xattr_access = 0;
	static u64 xattr_default = 0;
3597 3598
	int scanned = 0;

3599
	if (!xattr_access) {
3600 3601 3602 3603
		xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
					strlen(XATTR_NAME_POSIX_ACL_ACCESS));
		xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
					strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3604 3605
	}

3606
	slot++;
3607
	*first_xattr_slot = -1;
3608 3609 3610 3611 3612 3613 3614 3615
	while (slot < nritems) {
		btrfs_item_key_to_cpu(leaf, &found_key, slot);

		/* we found a different objectid, there must not be acls */
		if (found_key.objectid != objectid)
			return 0;

		/* we found an xattr, assume we've got an acl */
3616
		if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3617 3618
			if (*first_xattr_slot == -1)
				*first_xattr_slot = slot;
3619 3620 3621 3622
			if (found_key.offset == xattr_access ||
			    found_key.offset == xattr_default)
				return 1;
		}
3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646

		/*
		 * we found a key greater than an xattr key, there can't
		 * be any acls later on
		 */
		if (found_key.type > BTRFS_XATTR_ITEM_KEY)
			return 0;

		slot++;
		scanned++;

		/*
		 * it goes inode, inode backrefs, xattrs, extents,
		 * so if there are a ton of hard links to an inode there can
		 * be a lot of backrefs.  Don't waste time searching too hard,
		 * this is just an optimization
		 */
		if (scanned >= 8)
			break;
	}
	/* we hit the end of the leaf before we found an xattr or
	 * something larger than an xattr.  We have to assume the inode
	 * has acls
	 */
3647 3648
	if (*first_xattr_slot == -1)
		*first_xattr_slot = slot;
3649 3650 3651
	return 1;
}

3652 3653 3654
/*
 * read an inode from the btree into the in-memory inode
 */
3655
static int btrfs_read_locked_inode(struct inode *inode)
Chris Mason's avatar
Chris Mason committed
3656 3657
{
	struct btrfs_path *path;
3658
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
3659 3660 3661
	struct btrfs_inode_item *inode_item;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_key location;
3662
	unsigned long ptr;
3663
	int maybe_acls;
Josef Bacik's avatar
Josef Bacik committed
3664
	u32 rdev;
Chris Mason's avatar
Chris Mason committed
3665
	int ret;
3666
	bool filled = false;
3667
	int first_xattr_slot;
3668 3669 3670 3671

	ret = btrfs_fill_inode(inode, &rdev);
	if (!ret)
		filled = true;
Chris Mason's avatar
Chris Mason committed
3672 3673

	path = btrfs_alloc_path();
3674 3675
	if (!path) {
		ret = -ENOMEM;
3676
		goto make_bad;
3677
	}
3678

Chris Mason's avatar
Chris Mason committed
3679
	memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3680

Chris Mason's avatar
Chris Mason committed
3681
	ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3682 3683 3684
	if (ret) {
		if (ret > 0)
			ret = -ENOENT;
Chris Mason's avatar
Chris Mason committed
3685
		goto make_bad;
3686
	}
Chris Mason's avatar
Chris Mason committed
3687

3688
	leaf = path->nodes[0];
3689 3690

	if (filled)
3691
		goto cache_index;
3692

3693 3694 3695
	inode_item = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_inode_item);
	inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3696
	set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3697 3698
	i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
	i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3699
	btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
3700

3701 3702
	inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
	inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3703

3704 3705
	inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
	inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3706

3707 3708
	inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
	inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3709

3710 3711 3712 3713
	BTRFS_I(inode)->i_otime.tv_sec =
		btrfs_timespec_sec(leaf, &inode_item->otime);
	BTRFS_I(inode)->i_otime.tv_nsec =
		btrfs_timespec_nsec(leaf, &inode_item->otime);
3714

3715
	inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3716
	BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
Josef Bacik's avatar
Josef Bacik committed
3717 3718
	BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);

3719 3720 3721 3722 3723 3724 3725 3726 3727
	inode->i_version = btrfs_inode_sequence(leaf, inode_item);
	inode->i_generation = BTRFS_I(inode)->generation;
	inode->i_rdev = 0;
	rdev = btrfs_inode_rdev(leaf, inode_item);

	BTRFS_I(inode)->index_cnt = (u64)-1;
	BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);

cache_index:
Josef Bacik's avatar
Josef Bacik committed
3728 3729 3730 3731 3732
	/*
	 * If we were modified in the current generation and evicted from memory
	 * and then re-read we need to do a full sync since we don't have any
	 * idea about which extents were modified before we were evicted from
	 * cache.
3733 3734 3735
	 *
	 * This is required for both inode re-read from disk and delayed inode
	 * in delayed_nodes_tree.
Josef Bacik's avatar
Josef Bacik committed
3736 3737 3738 3739 3740
	 */
	if (BTRFS_I(inode)->last_trans == root->fs_info->generation)
		set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
			&BTRFS_I(inode)->runtime_flags);

3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763
	/*
	 * We don't persist the id of the transaction where an unlink operation
	 * against the inode was last made. So here we assume the inode might
	 * have been evicted, and therefore the exact value of last_unlink_trans
	 * lost, and set it to last_trans to avoid metadata inconsistencies
	 * between the inode and its parent if the inode is fsync'ed and the log
	 * replayed. For example, in the scenario:
	 *
	 * touch mydir/foo
	 * ln mydir/foo mydir/bar
	 * sync
	 * unlink mydir/bar
	 * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
	 * xfs_io -c fsync mydir/foo
	 * <power failure>
	 * mount fs, triggers fsync log replay
	 *
	 * We must make sure that when we fsync our inode foo we also log its
	 * parent inode, otherwise after log replay the parent still has the
	 * dentry with the "bar" name but our inode foo has a link count of 1
	 * and doesn't have an inode ref with the name "bar" anymore.
	 *
	 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3764
	 * but it guarantees correctness at the expense of occasional full
3765 3766 3767 3768 3769
	 * transaction commits on fsync if our inode is a directory, or if our
	 * inode is not a directory, logging its parent unnecessarily.
	 */
	BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;

3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791
	path->slots[0]++;
	if (inode->i_nlink != 1 ||
	    path->slots[0] >= btrfs_header_nritems(leaf))
		goto cache_acl;

	btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
	if (location.objectid != btrfs_ino(inode))
		goto cache_acl;

	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
	if (location.type == BTRFS_INODE_REF_KEY) {
		struct btrfs_inode_ref *ref;

		ref = (struct btrfs_inode_ref *)ptr;
		BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
	} else if (location.type == BTRFS_INODE_EXTREF_KEY) {
		struct btrfs_inode_extref *extref;

		extref = (struct btrfs_inode_extref *)ptr;
		BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
								     extref);
	}
3792
cache_acl:
3793 3794 3795 3796
	/*
	 * try to precache a NULL acl entry for files that don't have
	 * any xattrs or acls
	 */
3797
	maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3798 3799 3800 3801 3802 3803
					   btrfs_ino(inode), &first_xattr_slot);
	if (first_xattr_slot != -1) {
		path->slots[0] = first_xattr_slot;
		ret = btrfs_load_inode_props(inode, path);
		if (ret)
			btrfs_err(root->fs_info,
3804
				  "error loading props for ino %llu (root %llu): %d",
3805 3806 3807 3808 3809
				  btrfs_ino(inode),
				  root->root_key.objectid, ret);
	}
	btrfs_free_path(path);

3810 3811
	if (!maybe_acls)
		cache_no_acl(inode);
3812

Chris Mason's avatar
Chris Mason committed
3813 3814 3815
	switch (inode->i_mode & S_IFMT) {
	case S_IFREG:
		inode->i_mapping->a_ops = &btrfs_aops;
3816
		BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
Chris Mason's avatar
Chris Mason committed
3817 3818 3819 3820 3821
		inode->i_fop = &btrfs_file_operations;
		inode->i_op = &btrfs_file_inode_operations;
		break;
	case S_IFDIR:
		inode->i_fop = &btrfs_dir_file_operations;
3822
		inode->i_op = &btrfs_dir_inode_operations;
Chris Mason's avatar
Chris Mason committed
3823 3824 3825
		break;
	case S_IFLNK:
		inode->i_op = &btrfs_symlink_inode_operations;
3826
		inode_nohighmem(inode);
Chris Mason's avatar
Chris Mason committed
3827 3828
		inode->i_mapping->a_ops = &btrfs_symlink_aops;
		break;
Josef Bacik's avatar
Josef Bacik committed
3829
	default:
Jim Owens's avatar
Jim Owens committed
3830
		inode->i_op = &btrfs_special_inode_operations;
Josef Bacik's avatar
Josef Bacik committed
3831 3832
		init_special_inode(inode, inode->i_mode, rdev);
		break;
Chris Mason's avatar
Chris Mason committed
3833
	}
3834 3835

	btrfs_update_iflags(inode);
3836
	return 0;
Chris Mason's avatar
Chris Mason committed
3837 3838 3839 3840

make_bad:
	btrfs_free_path(path);
	make_bad_inode(inode);
3841
	return ret;
Chris Mason's avatar
Chris Mason committed
3842 3843
}

3844 3845 3846
/*
 * given a leaf and an inode, copy the inode fields into the leaf
 */
3847 3848
static void fill_inode_item(struct btrfs_trans_handle *trans,
			    struct extent_buffer *leaf,
3849
			    struct btrfs_inode_item *item,
Chris Mason's avatar
Chris Mason committed
3850 3851
			    struct inode *inode)
{
3852 3853 3854
	struct btrfs_map_token token;

	btrfs_init_map_token(&token);
3855

3856 3857 3858 3859 3860 3861
	btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
	btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
	btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
				   &token);
	btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
	btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3862

3863
	btrfs_set_token_timespec_sec(leaf, &item->atime,
3864
				     inode->i_atime.tv_sec, &token);
3865
	btrfs_set_token_timespec_nsec(leaf, &item->atime,
3866
				      inode->i_atime.tv_nsec, &token);
3867

3868
	btrfs_set_token_timespec_sec(leaf, &item->mtime,
3869
				     inode->i_mtime.tv_sec, &token);
3870
	btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3871
				      inode->i_mtime.tv_nsec, &token);
3872

3873
	btrfs_set_token_timespec_sec(leaf, &item->ctime,
3874
				     inode->i_ctime.tv_sec, &token);
3875
	btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3876
				      inode->i_ctime.tv_nsec, &token);
3877

3878 3879 3880 3881 3882
	btrfs_set_token_timespec_sec(leaf, &item->otime,
				     BTRFS_I(inode)->i_otime.tv_sec, &token);
	btrfs_set_token_timespec_nsec(leaf, &item->otime,
				      BTRFS_I(inode)->i_otime.tv_nsec, &token);

3883 3884 3885 3886 3887 3888 3889 3890 3891
	btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
				     &token);
	btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
					 &token);
	btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
	btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
	btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
	btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
	btrfs_set_token_inode_block_group(leaf, item, 0, &token);
Chris Mason's avatar
Chris Mason committed
3892 3893
}

3894 3895 3896
/*
 * copy everything in the in-memory inode into the btree.
 */
3897
static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3898
				struct btrfs_root *root, struct inode *inode)
Chris Mason's avatar
Chris Mason committed
3899 3900 3901
{
	struct btrfs_inode_item *inode_item;
	struct btrfs_path *path;
3902
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
3903 3904 3905
	int ret;

	path = btrfs_alloc_path();
3906 3907 3908
	if (!path)
		return -ENOMEM;

3909
	path->leave_spinning = 1;
3910 3911
	ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
				 1);
Chris Mason's avatar
Chris Mason committed
3912 3913 3914 3915 3916 3917
	if (ret) {
		if (ret > 0)
			ret = -ENOENT;
		goto failed;
	}

3918 3919
	leaf = path->nodes[0];
	inode_item = btrfs_item_ptr(leaf, path->slots[0],
3920
				    struct btrfs_inode_item);
Chris Mason's avatar
Chris Mason committed
3921

3922
	fill_inode_item(trans, leaf, inode_item, inode);
3923
	btrfs_mark_buffer_dirty(leaf);
3924
	btrfs_set_inode_last_trans(trans, inode);
Chris Mason's avatar
Chris Mason committed
3925 3926 3927 3928 3929 3930
	ret = 0;
failed:
	btrfs_free_path(path);
	return ret;
}

3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945
/*
 * copy everything in the in-memory inode into the btree.
 */
noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
				struct btrfs_root *root, struct inode *inode)
{
	int ret;

	/*
	 * If the inode is a free space inode, we can deadlock during commit
	 * if we put it into the delayed code.
	 *
	 * The data relocation inode should also be directly updated
	 * without delay
	 */
3946
	if (!btrfs_is_free_space_inode(inode)
3947
	    && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
3948
	    && !test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) {
3949 3950
		btrfs_update_root_times(trans, root);

3951 3952 3953 3954 3955 3956 3957 3958 3959
		ret = btrfs_delayed_update_inode(trans, root, inode);
		if (!ret)
			btrfs_set_inode_last_trans(trans, inode);
		return ret;
	}

	return btrfs_update_inode_item(trans, root, inode);
}

3960 3961 3962
noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
					 struct btrfs_root *root,
					 struct inode *inode)
3963 3964 3965 3966 3967 3968 3969 3970 3971
{
	int ret;

	ret = btrfs_update_inode(trans, root, inode);
	if (ret == -ENOSPC)
		return btrfs_update_inode_item(trans, root, inode);
	return ret;
}

3972 3973 3974 3975 3976
/*
 * unlink helper that gets used here in inode.c and in the tree logging
 * recovery code.  It remove a link in a directory with a given name, and
 * also drops the back refs in the inode to the directory
 */
3977 3978 3979 3980
static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct inode *dir, struct inode *inode,
				const char *name, int name_len)
Chris Mason's avatar
Chris Mason committed
3981 3982 3983
{
	struct btrfs_path *path;
	int ret = 0;
3984
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
3985
	struct btrfs_dir_item *di;
3986
	struct btrfs_key key;
3987
	u64 index;
3988 3989
	u64 ino = btrfs_ino(inode);
	u64 dir_ino = btrfs_ino(dir);
Chris Mason's avatar
Chris Mason committed
3990 3991

	path = btrfs_alloc_path();
3992 3993
	if (!path) {
		ret = -ENOMEM;
3994
		goto out;
3995 3996
	}

3997
	path->leave_spinning = 1;
3998
	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
Chris Mason's avatar
Chris Mason committed
3999 4000 4001 4002 4003 4004 4005 4006 4007
				    name, name_len, -1);
	if (IS_ERR(di)) {
		ret = PTR_ERR(di);
		goto err;
	}
	if (!di) {
		ret = -ENOENT;
		goto err;
	}
4008 4009
	leaf = path->nodes[0];
	btrfs_dir_item_key_to_cpu(leaf, di, &key);
Chris Mason's avatar
Chris Mason committed
4010
	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4011 4012
	if (ret)
		goto err;
4013
	btrfs_release_path(path);
Chris Mason's avatar
Chris Mason committed
4014

4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032
	/*
	 * If we don't have dir index, we have to get it by looking up
	 * the inode ref, since we get the inode ref, remove it directly,
	 * it is unnecessary to do delayed deletion.
	 *
	 * But if we have dir index, needn't search inode ref to get it.
	 * Since the inode ref is close to the inode item, it is better
	 * that we delay to delete it, and just do this deletion when
	 * we update the inode item.
	 */
	if (BTRFS_I(inode)->dir_index) {
		ret = btrfs_delayed_delete_inode_ref(inode);
		if (!ret) {
			index = BTRFS_I(inode)->dir_index;
			goto skip_backref;
		}
	}

4033 4034
	ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
				  dir_ino, &index);
4035
	if (ret) {
4036 4037
		btrfs_info(root->fs_info,
			"failed to delete reference to %.*s, inode %llu parent %llu",
4038
			name_len, name, ino, dir_ino);
4039
		btrfs_abort_transaction(trans, ret);
4040 4041
		goto err;
	}
4042
skip_backref:
4043
	ret = btrfs_delete_delayed_dir_index(trans, root, dir, index);
4044
	if (ret) {
4045
		btrfs_abort_transaction(trans, ret);
Chris Mason's avatar
Chris Mason committed
4046
		goto err;
4047
	}
Chris Mason's avatar
Chris Mason committed
4048

4049
	ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len,
4050
					 inode, dir_ino);
4051
	if (ret != 0 && ret != -ENOENT) {
4052
		btrfs_abort_transaction(trans, ret);
4053 4054
		goto err;
	}
4055 4056 4057

	ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len,
					   dir, index);
4058 4059
	if (ret == -ENOENT)
		ret = 0;
4060
	else if (ret)
4061
		btrfs_abort_transaction(trans, ret);
Chris Mason's avatar
Chris Mason committed
4062 4063
err:
	btrfs_free_path(path);
4064 4065 4066 4067
	if (ret)
		goto out;

	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
4068 4069
	inode_inc_iversion(inode);
	inode_inc_iversion(dir);
4070
	inode->i_ctime = dir->i_mtime =
4071
		dir->i_ctime = current_time(inode);
4072
	ret = btrfs_update_inode(trans, root, dir);
4073
out:
Chris Mason's avatar
Chris Mason committed
4074 4075 4076
	return ret;
}

4077 4078 4079 4080 4081 4082 4083 4084
int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
		       struct btrfs_root *root,
		       struct inode *dir, struct inode *inode,
		       const char *name, int name_len)
{
	int ret;
	ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
	if (!ret) {
Zach Brown's avatar
Zach Brown committed
4085
		drop_nlink(inode);
4086 4087 4088 4089
		ret = btrfs_update_inode(trans, root, inode);
	}
	return ret;
}
Chris Mason's avatar
Chris Mason committed
4090

4091 4092 4093
/*
 * helper to start transaction for unlink and rmdir.
 *
4094 4095 4096 4097
 * unlink and rmdir are special in btrfs, they do not always free space, so
 * if we cannot make our reservations the normal way try and see if there is
 * plenty of slack room in the global reserve to migrate, otherwise we cannot
 * allow the unlink to occur.
4098
 */
4099
static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4100
{
4101
	struct btrfs_root *root = BTRFS_I(dir)->root;
4102

4103 4104 4105 4106 4107 4108 4109
	/*
	 * 1 for the possible orphan item
	 * 1 for the dir item
	 * 1 for the dir index
	 * 1 for the inode ref
	 * 1 for the inode
	 */
4110
	return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4111 4112 4113 4114 4115 4116
}

static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
{
	struct btrfs_root *root = BTRFS_I(dir)->root;
	struct btrfs_trans_handle *trans;
4117
	struct inode *inode = d_inode(dentry);
4118 4119
	int ret;

4120
	trans = __unlink_start_trans(dir);
4121 4122
	if (IS_ERR(trans))
		return PTR_ERR(trans);
4123

4124
	btrfs_record_unlink_dir(trans, dir, d_inode(dentry), 0);
4125

4126
	ret = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4127
				 dentry->d_name.name, dentry->d_name.len);
4128 4129
	if (ret)
		goto out;
4130

4131
	if (inode->i_nlink == 0) {
4132
		ret = btrfs_orphan_add(trans, inode);
4133 4134
		if (ret)
			goto out;
4135
	}
4136

4137
out:
4138
	btrfs_end_transaction(trans, root);
4139
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
4140 4141 4142
	return ret;
}

4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153
int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct inode *dir, u64 objectid,
			const char *name, int name_len)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_dir_item *di;
	struct btrfs_key key;
	u64 index;
	int ret;
4154
	u64 dir_ino = btrfs_ino(dir);
4155 4156 4157 4158 4159

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

4160
	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4161
				   name, name_len, -1);
4162 4163 4164 4165 4166 4167 4168
	if (IS_ERR_OR_NULL(di)) {
		if (!di)
			ret = -ENOENT;
		else
			ret = PTR_ERR(di);
		goto out;
	}
4169 4170 4171 4172 4173

	leaf = path->nodes[0];
	btrfs_dir_item_key_to_cpu(leaf, di, &key);
	WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4174
	if (ret) {
4175
		btrfs_abort_transaction(trans, ret);
4176 4177
		goto out;
	}
4178
	btrfs_release_path(path);
4179 4180 4181

	ret = btrfs_del_root_ref(trans, root->fs_info->tree_root,
				 objectid, root->root_key.objectid,
4182
				 dir_ino, &index, name, name_len);
4183
	if (ret < 0) {
4184
		if (ret != -ENOENT) {
4185
			btrfs_abort_transaction(trans, ret);
4186 4187
			goto out;
		}
4188
		di = btrfs_search_dir_index_item(root, path, dir_ino,
4189
						 name, name_len);
4190 4191 4192 4193 4194
		if (IS_ERR_OR_NULL(di)) {
			if (!di)
				ret = -ENOENT;
			else
				ret = PTR_ERR(di);
4195
			btrfs_abort_transaction(trans, ret);
4196 4197
			goto out;
		}
4198 4199 4200

		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4201
		btrfs_release_path(path);
4202 4203
		index = key.offset;
	}
4204
	btrfs_release_path(path);
4205

4206
	ret = btrfs_delete_delayed_dir_index(trans, root, dir, index);
4207
	if (ret) {
4208
		btrfs_abort_transaction(trans, ret);
4209 4210
		goto out;
	}
4211 4212

	btrfs_i_size_write(dir, dir->i_size - name_len * 2);
4213
	inode_inc_iversion(dir);
4214
	dir->i_mtime = dir->i_ctime = current_time(dir);
4215
	ret = btrfs_update_inode_fallback(trans, root, dir);
4216
	if (ret)
4217
		btrfs_abort_transaction(trans, ret);
4218
out:
4219
	btrfs_free_path(path);
4220
	return ret;
4221 4222
}

Chris Mason's avatar
Chris Mason committed
4223 4224
static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
{
4225
	struct inode *inode = d_inode(dentry);
4226
	int err = 0;
Chris Mason's avatar
Chris Mason committed
4227 4228
	struct btrfs_root *root = BTRFS_I(dir)->root;
	struct btrfs_trans_handle *trans;
4229
	u64 last_unlink_trans;
Chris Mason's avatar
Chris Mason committed
4230

4231
	if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
Yan's avatar
Yan committed
4232
		return -ENOTEMPTY;
4233 4234
	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID)
		return -EPERM;
Yan's avatar
Yan committed
4235

4236
	trans = __unlink_start_trans(dir);
4237
	if (IS_ERR(trans))
4238 4239
		return PTR_ERR(trans);

4240
	if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4241 4242 4243 4244 4245 4246 4247
		err = btrfs_unlink_subvol(trans, root, dir,
					  BTRFS_I(inode)->location.objectid,
					  dentry->d_name.name,
					  dentry->d_name.len);
		goto out;
	}

4248 4249
	err = btrfs_orphan_add(trans, inode);
	if (err)
4250
		goto out;
4251

4252 4253
	last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;

Chris Mason's avatar
Chris Mason committed
4254
	/* now the directory is empty */
4255
	err = btrfs_unlink_inode(trans, root, dir, d_inode(dentry),
4256
				 dentry->d_name.name, dentry->d_name.len);
4257
	if (!err) {
4258
		btrfs_i_size_write(inode, 0);
4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272
		/*
		 * Propagate the last_unlink_trans value of the deleted dir to
		 * its parent directory. This is to prevent an unrecoverable
		 * log tree in the case we do something like this:
		 * 1) create dir foo
		 * 2) create snapshot under dir foo
		 * 3) delete the snapshot
		 * 4) rmdir foo
		 * 5) mkdir foo
		 * 6) fsync foo or some file inside foo
		 */
		if (last_unlink_trans >= trans->transid)
			BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
	}
4273
out:
4274
	btrfs_end_transaction(trans, root);
4275
	btrfs_btree_balance_dirty(root);
4276

Chris Mason's avatar
Chris Mason committed
4277 4278 4279
	return err;
}

4280 4281 4282 4283 4284 4285
static int truncate_space_check(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytes_deleted)
{
	int ret;

4286 4287 4288 4289
	/*
	 * This is only used to apply pressure to the enospc system, we don't
	 * intend to use this reservation at all.
	 */
4290
	bytes_deleted = btrfs_csum_bytes_to_leaves(root, bytes_deleted);
4291
	bytes_deleted *= root->nodesize;
4292 4293
	ret = btrfs_block_rsv_add(root, &root->fs_info->trans_block_rsv,
				  bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4294 4295 4296 4297
	if (!ret) {
		trace_btrfs_space_reservation(root->fs_info, "transaction",
					      trans->transid,
					      bytes_deleted, 1);
4298
		trans->bytes_reserved += bytes_deleted;
4299
	}
4300 4301 4302 4303
	return ret;

}

4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319
static int truncate_inline_extent(struct inode *inode,
				  struct btrfs_path *path,
				  struct btrfs_key *found_key,
				  const u64 item_end,
				  const u64 new_size)
{
	struct extent_buffer *leaf = path->nodes[0];
	int slot = path->slots[0];
	struct btrfs_file_extent_item *fi;
	u32 size = (u32)(new_size - found_key->offset);
	struct btrfs_root *root = BTRFS_I(inode)->root;

	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);

	if (btrfs_file_extent_compression(leaf, fi) != BTRFS_COMPRESS_NONE) {
		loff_t offset = new_size;
4320
		loff_t page_end = ALIGN(offset, PAGE_SIZE);
4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331

		/*
		 * Zero out the remaining of the last page of our inline extent,
		 * instead of directly truncating our inline extent here - that
		 * would be much more complex (decompressing all the data, then
		 * compressing the truncated data, which might be bigger than
		 * the size of the inline extent, resize the extent, etc).
		 * We release the path because to get the page we might need to
		 * read the extent item from disk (data not in the page cache).
		 */
		btrfs_release_path(path);
4332 4333
		return btrfs_truncate_block(inode, offset, page_end - offset,
					0);
4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345
	}

	btrfs_set_file_extent_ram_bytes(leaf, fi, size);
	size = btrfs_file_extent_calc_inline_size(size);
	btrfs_truncate_item(root, path, size, 1);

	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
		inode_sub_bytes(inode, item_end + 1 - new_size);

	return 0;
}

Chris Mason's avatar
Chris Mason committed
4346 4347 4348
/*
 * this can truncate away extent items, csum items and directory items.
 * It starts at a high offset and removes keys until it can't find
4349
 * any higher than new_size
Chris Mason's avatar
Chris Mason committed
4350 4351 4352
 *
 * csum items that cross the new i_size are truncated to the new size
 * as well.
4353 4354 4355
 *
 * min_type is the minimum key type to truncate down to.  If set to 0, this
 * will kill all the items on this inode, including the INODE_ITEM_KEY.
Chris Mason's avatar
Chris Mason committed
4356
 */
4357 4358 4359 4360
int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct inode *inode,
			       u64 new_size, u32 min_type)
Chris Mason's avatar
Chris Mason committed
4361 4362
{
	struct btrfs_path *path;
4363
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
4364
	struct btrfs_file_extent_item *fi;
4365 4366
	struct btrfs_key key;
	struct btrfs_key found_key;
Chris Mason's avatar
Chris Mason committed
4367
	u64 extent_start = 0;
4368
	u64 extent_num_bytes = 0;
4369
	u64 extent_offset = 0;
Chris Mason's avatar
Chris Mason committed
4370
	u64 item_end = 0;
4371
	u64 last_size = new_size;
4372
	u32 found_type = (u8)-1;
Chris Mason's avatar
Chris Mason committed
4373 4374
	int found_extent;
	int del_item;
4375 4376
	int pending_del_nr = 0;
	int pending_del_slot = 0;
4377
	int extent_type = -1;
4378 4379
	int ret;
	int err = 0;
4380
	u64 ino = btrfs_ino(inode);
4381
	u64 bytes_deleted = 0;
4382 4383
	bool be_nice = 0;
	bool should_throttle = 0;
4384
	bool should_end = 0;
4385 4386

	BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
Chris Mason's avatar
Chris Mason committed
4387

4388 4389 4390 4391 4392 4393 4394 4395
	/*
	 * for non-free space inodes and ref cows, we want to back off from
	 * time to time
	 */
	if (!btrfs_is_free_space_inode(inode) &&
	    test_bit(BTRFS_ROOT_REF_COWS, &root->state))
		be_nice = 1;

4396 4397 4398
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
4399
	path->reada = READA_BACK;
4400

Josef Bacik's avatar
Josef Bacik committed
4401 4402 4403 4404 4405
	/*
	 * We want to drop from the next block forward in case this new size is
	 * not block aligned since we will be keeping the last block of the
	 * extent just the way it is.
	 */
4406 4407
	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
	    root == root->fs_info->tree_root)
4408 4409
		btrfs_drop_extent_cache(inode, ALIGN(new_size,
					root->sectorsize), (u64)-1, 0);
4410

4411 4412 4413 4414 4415 4416 4417 4418 4419
	/*
	 * This function is also used to drop the items in the log tree before
	 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
	 * it is used to drop the loged items. So we shouldn't kill the delayed
	 * items.
	 */
	if (min_type == 0 && root == BTRFS_I(inode)->root)
		btrfs_kill_delayed_inode_items(inode);

4420
	key.objectid = ino;
Chris Mason's avatar
Chris Mason committed
4421
	key.offset = (u64)-1;
4422 4423
	key.type = (u8)-1;

4424
search_again:
4425 4426 4427 4428 4429
	/*
	 * with a 16K leaf size and 128MB extents, you can actually queue
	 * up a huge file in a single leaf.  Most of the time that
	 * bytes_deleted is > 0, it will be huge by the time we get here
	 */
4430
	if (be_nice && bytes_deleted > SZ_32M) {
4431 4432 4433 4434 4435 4436 4437
		if (btrfs_should_end_transaction(trans, root)) {
			err = -EAGAIN;
			goto error;
		}
	}


4438
	path->leave_spinning = 1;
4439
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4440 4441 4442 4443
	if (ret < 0) {
		err = ret;
		goto out;
	}
4444

4445
	if (ret > 0) {
4446 4447 4448
		/* there are no items in the tree for us to truncate, we're
		 * done
		 */
4449 4450
		if (path->slots[0] == 0)
			goto out;
4451 4452 4453
		path->slots[0]--;
	}

4454
	while (1) {
Chris Mason's avatar
Chris Mason committed
4455
		fi = NULL;
4456 4457
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4458
		found_type = found_key.type;
Chris Mason's avatar
Chris Mason committed
4459

4460
		if (found_key.objectid != ino)
Chris Mason's avatar
Chris Mason committed
4461
			break;
4462

4463
		if (found_type < min_type)
Chris Mason's avatar
Chris Mason committed
4464 4465
			break;

4466
		item_end = found_key.offset;
Chris Mason's avatar
Chris Mason committed
4467
		if (found_type == BTRFS_EXTENT_DATA_KEY) {
4468
			fi = btrfs_item_ptr(leaf, path->slots[0],
Chris Mason's avatar
Chris Mason committed
4469
					    struct btrfs_file_extent_item);
4470 4471
			extent_type = btrfs_file_extent_type(leaf, fi);
			if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4472
				item_end +=
4473
				    btrfs_file_extent_num_bytes(leaf, fi);
4474 4475
			} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
				item_end += btrfs_file_extent_inline_len(leaf,
4476
							 path->slots[0], fi);
Chris Mason's avatar
Chris Mason committed
4477
			}
4478
			item_end--;
Chris Mason's avatar
Chris Mason committed
4479
		}
4480 4481 4482
		if (found_type > min_type) {
			del_item = 1;
		} else {
4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493
			if (item_end < new_size) {
				/*
				 * With NO_HOLES mode, for the following mapping
				 *
				 * [0-4k][hole][8k-12k]
				 *
				 * if truncating isize down to 6k, it ends up
				 * isize being 8k.
				 */
				if (btrfs_fs_incompat(root->fs_info, NO_HOLES))
					last_size = new_size;
4494
				break;
4495
			}
4496 4497 4498 4499
			if (found_key.offset >= new_size)
				del_item = 1;
			else
				del_item = 0;
Chris Mason's avatar
Chris Mason committed
4500 4501 4502
		}
		found_extent = 0;
		/* FIXME, shrink the extent if the ref count is only 1 */
4503 4504 4505
		if (found_type != BTRFS_EXTENT_DATA_KEY)
			goto delete;

4506 4507 4508 4509 4510
		if (del_item)
			last_size = found_key.offset;
		else
			last_size = new_size;

4511
		if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
Chris Mason's avatar
Chris Mason committed
4512
			u64 num_dec;
4513
			extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4514
			if (!del_item) {
4515 4516
				u64 orig_num_bytes =
					btrfs_file_extent_num_bytes(leaf, fi);
4517 4518 4519
				extent_num_bytes = ALIGN(new_size -
						found_key.offset,
						root->sectorsize);
4520 4521 4522
				btrfs_set_file_extent_num_bytes(leaf, fi,
							 extent_num_bytes);
				num_dec = (orig_num_bytes -
Chris Mason's avatar
Chris Mason committed
4523
					   extent_num_bytes);
4524 4525 4526
				if (test_bit(BTRFS_ROOT_REF_COWS,
					     &root->state) &&
				    extent_start != 0)
4527
					inode_sub_bytes(inode, num_dec);
4528
				btrfs_mark_buffer_dirty(leaf);
Chris Mason's avatar
Chris Mason committed
4529
			} else {
4530 4531 4532
				extent_num_bytes =
					btrfs_file_extent_disk_num_bytes(leaf,
									 fi);
4533 4534 4535
				extent_offset = found_key.offset -
					btrfs_file_extent_offset(leaf, fi);

Chris Mason's avatar
Chris Mason committed
4536
				/* FIXME blocksize != 4096 */
Chris Mason's avatar
Chris Mason committed
4537
				num_dec = btrfs_file_extent_num_bytes(leaf, fi);
Chris Mason's avatar
Chris Mason committed
4538 4539
				if (extent_start != 0) {
					found_extent = 1;
4540 4541
					if (test_bit(BTRFS_ROOT_REF_COWS,
						     &root->state))
4542
						inode_sub_bytes(inode, num_dec);
4543
				}
Chris Mason's avatar
Chris Mason committed
4544
			}
Chris Mason's avatar
Chris Mason committed
4545
		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4546 4547 4548 4549 4550 4551 4552
			/*
			 * we can't truncate inline items that have had
			 * special encodings
			 */
			if (!del_item &&
			    btrfs_file_extent_encryption(leaf, fi) == 0 &&
			    btrfs_file_extent_other_encoding(leaf, fi) == 0) {
4553 4554

				/*
4555 4556 4557
				 * Need to release path in order to truncate a
				 * compressed extent. So delete any accumulated
				 * extent items so far.
4558
				 */
4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576
				if (btrfs_file_extent_compression(leaf, fi) !=
				    BTRFS_COMPRESS_NONE && pending_del_nr) {
					err = btrfs_del_items(trans, root, path,
							      pending_del_slot,
							      pending_del_nr);
					if (err) {
						btrfs_abort_transaction(trans,
									err);
						goto error;
					}
					pending_del_nr = 0;
				}

				err = truncate_inline_extent(inode, path,
							     &found_key,
							     item_end,
							     new_size);
				if (err) {
4577
					btrfs_abort_transaction(trans, err);
4578 4579
					goto error;
				}
4580 4581
			} else if (test_bit(BTRFS_ROOT_REF_COWS,
					    &root->state)) {
4582
				inode_sub_bytes(inode, item_end + 1 - new_size);
Chris Mason's avatar
Chris Mason committed
4583
			}
Chris Mason's avatar
Chris Mason committed
4584
		}
4585
delete:
Chris Mason's avatar
Chris Mason committed
4586
		if (del_item) {
4587 4588 4589 4590 4591 4592 4593 4594 4595 4596
			if (!pending_del_nr) {
				/* no pending yet, add ourselves */
				pending_del_slot = path->slots[0];
				pending_del_nr = 1;
			} else if (pending_del_nr &&
				   path->slots[0] + 1 == pending_del_slot) {
				/* hop on the pending chunk */
				pending_del_nr++;
				pending_del_slot = path->slots[0];
			} else {
4597
				BUG();
4598
			}
Chris Mason's avatar
Chris Mason committed
4599 4600 4601
		} else {
			break;
		}
4602 4603
		should_throttle = 0;

4604 4605 4606
		if (found_extent &&
		    (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
		     root == root->fs_info->tree_root)) {
4607
			btrfs_set_path_blocking(path);
4608
			bytes_deleted += extent_num_bytes;
Chris Mason's avatar
Chris Mason committed
4609
			ret = btrfs_free_extent(trans, root, extent_start,
4610 4611
						extent_num_bytes, 0,
						btrfs_header_owner(leaf),
4612
						ino, extent_offset);
Chris Mason's avatar
Chris Mason committed
4613
			BUG_ON(ret);
4614
			if (btrfs_should_throttle_delayed_refs(trans, root))
4615
				btrfs_async_run_delayed_refs(root,
4616 4617
					trans->delayed_ref_updates * 2,
					trans->transid, 0);
4618 4619 4620 4621 4622 4623 4624 4625 4626 4627
			if (be_nice) {
				if (truncate_space_check(trans, root,
							 extent_num_bytes)) {
					should_end = 1;
				}
				if (btrfs_should_throttle_delayed_refs(trans,
								       root)) {
					should_throttle = 1;
				}
			}
Chris Mason's avatar
Chris Mason committed
4628
		}
4629

4630 4631 4632 4633
		if (found_type == BTRFS_INODE_ITEM_KEY)
			break;

		if (path->slots[0] == 0 ||
4634
		    path->slots[0] != pending_del_slot ||
4635
		    should_throttle || should_end) {
4636 4637 4638 4639
			if (pending_del_nr) {
				ret = btrfs_del_items(trans, root, path,
						pending_del_slot,
						pending_del_nr);
4640
				if (ret) {
4641
					btrfs_abort_transaction(trans, ret);
4642 4643
					goto error;
				}
4644 4645
				pending_del_nr = 0;
			}
4646
			btrfs_release_path(path);
4647
			if (should_throttle) {
4648 4649 4650 4651 4652 4653 4654 4655
				unsigned long updates = trans->delayed_ref_updates;
				if (updates) {
					trans->delayed_ref_updates = 0;
					ret = btrfs_run_delayed_refs(trans, root, updates * 2);
					if (ret && !err)
						err = ret;
				}
			}
4656 4657 4658 4659 4660 4661 4662 4663
			/*
			 * if we failed to refill our space rsv, bail out
			 * and let the transaction restart
			 */
			if (should_end) {
				err = -EAGAIN;
				goto error;
			}
4664
			goto search_again;
4665 4666
		} else {
			path->slots[0]--;
4667
		}
Chris Mason's avatar
Chris Mason committed
4668
	}
4669
out:
4670 4671 4672
	if (pending_del_nr) {
		ret = btrfs_del_items(trans, root, path, pending_del_slot,
				      pending_del_nr);
4673
		if (ret)
4674
			btrfs_abort_transaction(trans, ret);
4675
	}
4676
error:
4677
	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID)
4678
		btrfs_ordered_update_i_size(inode, last_size, NULL);
4679

Chris Mason's avatar
Chris Mason committed
4680
	btrfs_free_path(path);
4681

4682
	if (be_nice && bytes_deleted > SZ_32M) {
4683 4684 4685 4686 4687 4688 4689 4690
		unsigned long updates = trans->delayed_ref_updates;
		if (updates) {
			trans->delayed_ref_updates = 0;
			ret = btrfs_run_delayed_refs(trans, root, updates * 2);
			if (ret && !err)
				err = ret;
		}
	}
4691
	return err;
Chris Mason's avatar
Chris Mason committed
4692 4693 4694
}

/*
4695
 * btrfs_truncate_block - read, zero a chunk and write a block
Josef Bacik's avatar
Josef Bacik committed
4696 4697 4698 4699 4700 4701
 * @inode - inode that we're zeroing
 * @from - the offset to start zeroing
 * @len - the length to zero, 0 to zero the entire range respective to the
 *	offset
 * @front - zero up to the offset instead of from the offset on
 *
4702
 * This will find the block for the "from" offset and cow the block and zero the
Josef Bacik's avatar
Josef Bacik committed
4703
 * part we want to zero.  This is used with truncate and hole punching.
Chris Mason's avatar
Chris Mason committed
4704
 */
4705
int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
Josef Bacik's avatar
Josef Bacik committed
4706
			int front)
Chris Mason's avatar
Chris Mason committed
4707
{
Josef Bacik's avatar
Josef Bacik committed
4708
	struct address_space *mapping = inode->i_mapping;
4709
	struct btrfs_root *root = BTRFS_I(inode)->root;
4710 4711
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
	struct btrfs_ordered_extent *ordered;
4712
	struct extent_state *cached_state = NULL;
4713
	char *kaddr;
4714
	u32 blocksize = root->sectorsize;
4715
	pgoff_t index = from >> PAGE_SHIFT;
4716
	unsigned offset = from & (blocksize - 1);
Chris Mason's avatar
Chris Mason committed
4717
	struct page *page;
4718
	gfp_t mask = btrfs_alloc_write_mask(mapping);
Chris Mason's avatar
Chris Mason committed
4719
	int ret = 0;
4720 4721
	u64 block_start;
	u64 block_end;
Chris Mason's avatar
Chris Mason committed
4722

Josef Bacik's avatar
Josef Bacik committed
4723 4724
	if ((offset & (blocksize - 1)) == 0 &&
	    (!len || ((len & (blocksize - 1)) == 0)))
Chris Mason's avatar
Chris Mason committed
4725
		goto out;
4726

4727
	ret = btrfs_delalloc_reserve_space(inode,
4728
			round_down(from, blocksize), blocksize);
4729 4730
	if (ret)
		goto out;
Chris Mason's avatar
Chris Mason committed
4731

4732
again:
4733
	page = find_or_create_page(mapping, index, mask);
4734
	if (!page) {
4735
		btrfs_delalloc_release_space(inode,
4736 4737
				round_down(from, blocksize),
				blocksize);
4738
		ret = -ENOMEM;
Chris Mason's avatar
Chris Mason committed
4739
		goto out;
4740
	}
4741

4742 4743
	block_start = round_down(from, blocksize);
	block_end = block_start + blocksize - 1;
4744

Chris Mason's avatar
Chris Mason committed
4745
	if (!PageUptodate(page)) {
4746
		ret = btrfs_readpage(NULL, page);
Chris Mason's avatar
Chris Mason committed
4747
		lock_page(page);
4748 4749
		if (page->mapping != mapping) {
			unlock_page(page);
4750
			put_page(page);
4751 4752
			goto again;
		}
Chris Mason's avatar
Chris Mason committed
4753 4754
		if (!PageUptodate(page)) {
			ret = -EIO;
4755
			goto out_unlock;
Chris Mason's avatar
Chris Mason committed
4756 4757
		}
	}
4758
	wait_on_page_writeback(page);
4759

4760
	lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4761 4762
	set_page_extent_mapped(page);

4763
	ordered = btrfs_lookup_ordered_extent(inode, block_start);
4764
	if (ordered) {
4765
		unlock_extent_cached(io_tree, block_start, block_end,
4766
				     &cached_state, GFP_NOFS);
4767
		unlock_page(page);
4768
		put_page(page);
4769
		btrfs_start_ordered_extent(inode, ordered, 1);
4770 4771 4772 4773
		btrfs_put_ordered_extent(ordered);
		goto again;
	}

4774
	clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4775 4776
			  EXTENT_DIRTY | EXTENT_DELALLOC |
			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4777
			  0, 0, &cached_state, GFP_NOFS);
4778

4779
	ret = btrfs_set_extent_delalloc(inode, block_start, block_end,
4780
					&cached_state, 0);
Josef Bacik's avatar
Josef Bacik committed
4781
	if (ret) {
4782
		unlock_extent_cached(io_tree, block_start, block_end,
4783
				     &cached_state, GFP_NOFS);
Josef Bacik's avatar
Josef Bacik committed
4784 4785 4786
		goto out_unlock;
	}

4787
	if (offset != blocksize) {
Josef Bacik's avatar
Josef Bacik committed
4788
		if (!len)
4789
			len = blocksize - offset;
4790
		kaddr = kmap(page);
Josef Bacik's avatar
Josef Bacik committed
4791
		if (front)
4792 4793
			memset(kaddr + (block_start - page_offset(page)),
				0, offset);
Josef Bacik's avatar
Josef Bacik committed
4794
		else
4795 4796
			memset(kaddr + (block_start - page_offset(page)) +  offset,
				0, len);
4797 4798 4799
		flush_dcache_page(page);
		kunmap(page);
	}
4800
	ClearPageChecked(page);
4801
	set_page_dirty(page);
4802
	unlock_extent_cached(io_tree, block_start, block_end, &cached_state,
4803
			     GFP_NOFS);
Chris Mason's avatar
Chris Mason committed
4804

4805
out_unlock:
4806
	if (ret)
4807 4808
		btrfs_delalloc_release_space(inode, block_start,
					     blocksize);
Chris Mason's avatar
Chris Mason committed
4809
	unlock_page(page);
4810
	put_page(page);
Chris Mason's avatar
Chris Mason committed
4811 4812 4813 4814
out:
	return ret;
}

4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842
static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
			     u64 offset, u64 len)
{
	struct btrfs_trans_handle *trans;
	int ret;

	/*
	 * Still need to make sure the inode looks like it's been updated so
	 * that any holes get logged if we fsync.
	 */
	if (btrfs_fs_incompat(root->fs_info, NO_HOLES)) {
		BTRFS_I(inode)->last_trans = root->fs_info->generation;
		BTRFS_I(inode)->last_sub_trans = root->log_transid;
		BTRFS_I(inode)->last_log_commit = root->last_log_commit;
		return 0;
	}

	/*
	 * 1 - for the one we're dropping
	 * 1 - for the one we're adding
	 * 1 - for updating the inode.
	 */
	trans = btrfs_start_transaction(root, 3);
	if (IS_ERR(trans))
		return PTR_ERR(trans);

	ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
	if (ret) {
4843
		btrfs_abort_transaction(trans, ret);
4844 4845 4846 4847 4848 4849 4850
		btrfs_end_transaction(trans, root);
		return ret;
	}

	ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), offset,
				       0, 0, len, 0, len, 0, 0, 0);
	if (ret)
4851
		btrfs_abort_transaction(trans, ret);
4852 4853 4854 4855 4856 4857
	else
		btrfs_update_inode(trans, root, inode);
	btrfs_end_transaction(trans, root);
	return ret;
}

4858 4859 4860 4861 4862 4863
/*
 * This function puts in dummy file extents for the area we're creating a hole
 * for.  So if we are truncating this file to a larger size we need to insert
 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
 * the range between oldsize and size
 */
4864
int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
Chris Mason's avatar
Chris Mason committed
4865
{
Yan Zheng's avatar
Yan Zheng committed
4866 4867
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4868
	struct extent_map *em = NULL;
4869
	struct extent_state *cached_state = NULL;
Josef Bacik's avatar
Josef Bacik committed
4870
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4871 4872
	u64 hole_start = ALIGN(oldsize, root->sectorsize);
	u64 block_end = ALIGN(size, root->sectorsize);
Yan Zheng's avatar
Yan Zheng committed
4873 4874 4875
	u64 last_byte;
	u64 cur_offset;
	u64 hole_size;
Josef Bacik's avatar
Josef Bacik committed
4876
	int err = 0;
Chris Mason's avatar
Chris Mason committed
4877

4878
	/*
4879 4880
	 * If our size started in the middle of a block we need to zero out the
	 * rest of the block before we expand the i_size, otherwise we could
4881 4882
	 * expose stale data.
	 */
4883
	err = btrfs_truncate_block(inode, oldsize, 0, 0);
4884 4885 4886
	if (err)
		return err;

Yan Zheng's avatar
Yan Zheng committed
4887 4888 4889 4890 4891
	if (size <= hole_start)
		return 0;

	while (1) {
		struct btrfs_ordered_extent *ordered;
4892

4893
		lock_extent_bits(io_tree, hole_start, block_end - 1,
4894
				 &cached_state);
4895 4896
		ordered = btrfs_lookup_ordered_range(inode, hole_start,
						     block_end - hole_start);
Yan Zheng's avatar
Yan Zheng committed
4897 4898
		if (!ordered)
			break;
4899 4900
		unlock_extent_cached(io_tree, hole_start, block_end - 1,
				     &cached_state, GFP_NOFS);
4901
		btrfs_start_ordered_extent(inode, ordered, 1);
Yan Zheng's avatar
Yan Zheng committed
4902 4903
		btrfs_put_ordered_extent(ordered);
	}
Chris Mason's avatar
Chris Mason committed
4904

Yan Zheng's avatar
Yan Zheng committed
4905 4906 4907 4908
	cur_offset = hole_start;
	while (1) {
		em = btrfs_get_extent(inode, NULL, 0, cur_offset,
				block_end - cur_offset, 0);
4909 4910
		if (IS_ERR(em)) {
			err = PTR_ERR(em);
4911
			em = NULL;
4912 4913
			break;
		}
Yan Zheng's avatar
Yan Zheng committed
4914
		last_byte = min(extent_map_end(em), block_end);
4915
		last_byte = ALIGN(last_byte , root->sectorsize);
4916
		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
Josef Bacik's avatar
Josef Bacik committed
4917
			struct extent_map *hole_em;
Yan Zheng's avatar
Yan Zheng committed
4918
			hole_size = last_byte - cur_offset;
Josef Bacik's avatar
Josef Bacik committed
4919

4920 4921 4922
			err = maybe_insert_hole(root, inode, cur_offset,
						hole_size);
			if (err)
4923
				break;
Josef Bacik's avatar
Josef Bacik committed
4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934
			btrfs_drop_extent_cache(inode, cur_offset,
						cur_offset + hole_size - 1, 0);
			hole_em = alloc_extent_map();
			if (!hole_em) {
				set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
					&BTRFS_I(inode)->runtime_flags);
				goto next;
			}
			hole_em->start = cur_offset;
			hole_em->len = hole_size;
			hole_em->orig_start = cur_offset;
4935

Josef Bacik's avatar
Josef Bacik committed
4936 4937
			hole_em->block_start = EXTENT_MAP_HOLE;
			hole_em->block_len = 0;
4938
			hole_em->orig_block_len = 0;
Josef Bacik's avatar
Josef Bacik committed
4939
			hole_em->ram_bytes = hole_size;
Josef Bacik's avatar
Josef Bacik committed
4940 4941
			hole_em->bdev = root->fs_info->fs_devices->latest_bdev;
			hole_em->compress_type = BTRFS_COMPRESS_NONE;
4942
			hole_em->generation = root->fs_info->generation;
4943

Josef Bacik's avatar
Josef Bacik committed
4944 4945
			while (1) {
				write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
4946
				err = add_extent_mapping(em_tree, hole_em, 1);
Josef Bacik's avatar
Josef Bacik committed
4947 4948 4949 4950 4951 4952 4953 4954
				write_unlock(&em_tree->lock);
				if (err != -EEXIST)
					break;
				btrfs_drop_extent_cache(inode, cur_offset,
							cur_offset +
							hole_size - 1, 0);
			}
			free_extent_map(hole_em);
Yan Zheng's avatar
Yan Zheng committed
4955
		}
4956
next:
Yan Zheng's avatar
Yan Zheng committed
4957
		free_extent_map(em);
4958
		em = NULL;
Yan Zheng's avatar
Yan Zheng committed
4959
		cur_offset = last_byte;
4960
		if (cur_offset >= block_end)
Yan Zheng's avatar
Yan Zheng committed
4961 4962
			break;
	}
4963
	free_extent_map(em);
4964 4965
	unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
			     GFP_NOFS);
Yan Zheng's avatar
Yan Zheng committed
4966 4967
	return err;
}
Chris Mason's avatar
Chris Mason committed
4968

4969
static int btrfs_setsize(struct inode *inode, struct iattr *attr)
4970
{
4971 4972
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_trans_handle *trans;
4973
	loff_t oldsize = i_size_read(inode);
4974 4975
	loff_t newsize = attr->ia_size;
	int mask = attr->ia_valid;
4976 4977
	int ret;

4978 4979 4980 4981 4982 4983
	/*
	 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
	 * special case where we need to update the times despite not having
	 * these flags set.  For all other operations the VFS set these flags
	 * explicitly if it wants a timestamp update.
	 */
4984 4985 4986 4987
	if (newsize != oldsize) {
		inode_inc_iversion(inode);
		if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
			inode->i_ctime = inode->i_mtime =
4988
				current_time(inode);
4989
	}
4990

4991
	if (newsize > oldsize) {
4992 4993 4994 4995 4996 4997 4998
		/*
		 * Don't do an expanding truncate while snapshoting is ongoing.
		 * This is to ensure the snapshot captures a fully consistent
		 * state of this file - if the snapshot captures this expanding
		 * truncation, it must capture all writes that happened before
		 * this truncation.
		 */
4999
		btrfs_wait_for_snapshot_creation(root);
5000
		ret = btrfs_cont_expand(inode, oldsize, newsize);
5001 5002
		if (ret) {
			btrfs_end_write_no_snapshoting(root);
5003
			return ret;
5004
		}
5005

5006
		trans = btrfs_start_transaction(root, 1);
5007 5008
		if (IS_ERR(trans)) {
			btrfs_end_write_no_snapshoting(root);
5009
			return PTR_ERR(trans);
5010
		}
5011 5012 5013

		i_size_write(inode, newsize);
		btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5014
		pagecache_isize_extended(inode, oldsize, newsize);
5015
		ret = btrfs_update_inode(trans, root, inode);
5016
		btrfs_end_write_no_snapshoting(root);
5017
		btrfs_end_transaction(trans, root);
5018
	} else {
5019

5020 5021 5022 5023 5024 5025
		/*
		 * We're truncating a file that used to have good data down to
		 * zero. Make sure it gets into the ordered flush list so that
		 * any new writes get down to disk quickly.
		 */
		if (newsize == 0)
5026 5027
			set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
				&BTRFS_I(inode)->runtime_flags);
5028

5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043
		/*
		 * 1 for the orphan item we're going to add
		 * 1 for the orphan item deletion.
		 */
		trans = btrfs_start_transaction(root, 2);
		if (IS_ERR(trans))
			return PTR_ERR(trans);

		/*
		 * We need to do this in case we fail at _any_ point during the
		 * actual truncate.  Once we do the truncate_setsize we could
		 * invalidate pages which forces any outstanding ordered io to
		 * be instantly completed which will give us extents that need
		 * to be truncated.  If we fail to get an orphan inode down we
		 * could have left over extents that were never meant to live,
5044
		 * so we need to guarantee from this point on that everything
5045 5046 5047 5048 5049 5050 5051
		 * will be consistent.
		 */
		ret = btrfs_orphan_add(trans, inode);
		btrfs_end_transaction(trans, root);
		if (ret)
			return ret;

5052 5053
		/* we don't support swapfiles, so vmtruncate shouldn't fail */
		truncate_setsize(inode, newsize);
5054 5055 5056 5057 5058 5059

		/* Disable nonlocked read DIO to avoid the end less truncate */
		btrfs_inode_block_unlocked_dio(inode);
		inode_dio_wait(inode);
		btrfs_inode_resume_unlocked_dio(inode);

5060
		ret = btrfs_truncate(inode);
5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077
		if (ret && inode->i_nlink) {
			int err;

			/*
			 * failed to truncate, disk_i_size is only adjusted down
			 * as we remove extents, so it should represent the true
			 * size of the inode, so reset the in memory size and
			 * delete our orphan entry.
			 */
			trans = btrfs_join_transaction(root);
			if (IS_ERR(trans)) {
				btrfs_orphan_del(NULL, inode);
				return ret;
			}
			i_size_write(inode, BTRFS_I(inode)->disk_i_size);
			err = btrfs_orphan_del(trans, inode);
			if (err)
5078
				btrfs_abort_transaction(trans, err);
5079 5080
			btrfs_end_transaction(trans, root);
		}
5081 5082
	}

5083
	return ret;
5084 5085
}

Yan Zheng's avatar
Yan Zheng committed
5086 5087
static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
{
5088
	struct inode *inode = d_inode(dentry);
5089
	struct btrfs_root *root = BTRFS_I(inode)->root;
Yan Zheng's avatar
Yan Zheng committed
5090
	int err;
Chris Mason's avatar
Chris Mason committed
5091

5092 5093 5094
	if (btrfs_root_readonly(root))
		return -EROFS;

5095
	err = setattr_prepare(dentry, attr);
Yan Zheng's avatar
Yan Zheng committed
5096 5097
	if (err)
		return err;
Chris Mason's avatar
Chris Mason committed
5098

5099
	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5100
		err = btrfs_setsize(inode, attr);
5101 5102
		if (err)
			return err;
Chris Mason's avatar
Chris Mason committed
5103
	}
Yan Zheng's avatar
Yan Zheng committed
5104

Christoph Hellwig's avatar
Christoph Hellwig committed
5105 5106
	if (attr->ia_valid) {
		setattr_copy(inode, attr);
5107
		inode_inc_iversion(inode);
5108
		err = btrfs_dirty_inode(inode);
Christoph Hellwig's avatar
Christoph Hellwig committed
5109

5110
		if (!err && attr->ia_valid & ATTR_MODE)
5111
			err = posix_acl_chmod(inode, inode->i_mode);
Christoph Hellwig's avatar
Christoph Hellwig committed
5112
	}
Josef Bacik's avatar
Josef Bacik committed
5113

Chris Mason's avatar
Chris Mason committed
5114 5115
	return err;
}
5116

5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135
/*
 * While truncating the inode pages during eviction, we get the VFS calling
 * btrfs_invalidatepage() against each page of the inode. This is slow because
 * the calls to btrfs_invalidatepage() result in a huge amount of calls to
 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
 * extent_state structures over and over, wasting lots of time.
 *
 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
 * those expensive operations on a per page basis and do only the ordered io
 * finishing, while we release here the extent_map and extent_state structures,
 * without the excessive merging and splitting.
 */
static void evict_inode_truncate_pages(struct inode *inode)
{
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
	struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
	struct rb_node *node;

	ASSERT(inode->i_state & I_FREEING);
5136
	truncate_inode_pages_final(&inode->i_data);
5137 5138 5139 5140 5141 5142 5143

	write_lock(&map_tree->lock);
	while (!RB_EMPTY_ROOT(&map_tree->map)) {
		struct extent_map *em;

		node = rb_first(&map_tree->map);
		em = rb_entry(node, struct extent_map, rb_node);
5144 5145
		clear_bit(EXTENT_FLAG_PINNED, &em->flags);
		clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5146 5147
		remove_extent_mapping(map_tree, em);
		free_extent_map(em);
5148 5149 5150 5151 5152
		if (need_resched()) {
			write_unlock(&map_tree->lock);
			cond_resched();
			write_lock(&map_tree->lock);
		}
5153 5154 5155
	}
	write_unlock(&map_tree->lock);

5156 5157 5158
	/*
	 * Keep looping until we have no more ranges in the io tree.
	 * We can have ongoing bios started by readpages (called from readahead)
5159 5160 5161
	 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
	 * still in progress (unlocked the pages in the bio but did not yet
	 * unlocked the ranges in the io tree). Therefore this means some
5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
	 * ranges can still be locked and eviction started because before
	 * submitting those bios, which are executed by a separate task (work
	 * queue kthread), inode references (inode->i_count) were not taken
	 * (which would be dropped in the end io callback of each bio).
	 * Therefore here we effectively end up waiting for those bios and
	 * anyone else holding locked ranges without having bumped the inode's
	 * reference count - if we don't do it, when they access the inode's
	 * io_tree to unlock a range it may be too late, leading to an
	 * use-after-free issue.
	 */
5172 5173 5174 5175
	spin_lock(&io_tree->lock);
	while (!RB_EMPTY_ROOT(&io_tree->state)) {
		struct extent_state *state;
		struct extent_state *cached_state = NULL;
5176 5177
		u64 start;
		u64 end;
5178 5179 5180

		node = rb_first(&io_tree->state);
		state = rb_entry(node, struct extent_state, rb_node);
5181 5182
		start = state->start;
		end = state->end;
5183 5184
		spin_unlock(&io_tree->lock);

5185
		lock_extent_bits(io_tree, start, end, &cached_state);
5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197

		/*
		 * If still has DELALLOC flag, the extent didn't reach disk,
		 * and its reserved space won't be freed by delayed_ref.
		 * So we need to free its reserved space here.
		 * (Refer to comment in btrfs_invalidatepage, case 2)
		 *
		 * Note, end is the bytenr of last byte, so we need + 1 here.
		 */
		if (state->state & EXTENT_DELALLOC)
			btrfs_qgroup_free_data(inode, start, end - start + 1);

5198
		clear_extent_bit(io_tree, start, end,
5199 5200 5201 5202 5203
				 EXTENT_LOCKED | EXTENT_DIRTY |
				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
				 EXTENT_DEFRAG, 1, 1,
				 &cached_state, GFP_NOFS);

5204
		cond_resched();
5205 5206 5207 5208 5209
		spin_lock(&io_tree->lock);
	}
	spin_unlock(&io_tree->lock);
}

Al Viro's avatar
Al Viro committed
5210
void btrfs_evict_inode(struct inode *inode)
Chris Mason's avatar
Chris Mason committed
5211 5212 5213
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(inode)->root;
5214
	struct btrfs_block_rsv *rsv, *global_rsv;
5215
	int steal_from_global = 0;
5216
	u64 min_size;
Chris Mason's avatar
Chris Mason committed
5217 5218
	int ret;

5219 5220
	trace_btrfs_inode_evict(inode);

5221 5222 5223 5224 5225 5226 5227
	if (!root) {
		kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
		return;
	}

	min_size = btrfs_calc_trunc_metadata_size(root, 1);

5228 5229
	evict_inode_truncate_pages(inode);

5230 5231 5232 5233
	if (inode->i_nlink &&
	    ((btrfs_root_refs(&root->root_item) != 0 &&
	      root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
	     btrfs_is_free_space_inode(inode)))
Al Viro's avatar
Al Viro committed
5234 5235
		goto no_delete;

Chris Mason's avatar
Chris Mason committed
5236
	if (is_bad_inode(inode)) {
5237
		btrfs_orphan_del(NULL, inode);
Chris Mason's avatar
Chris Mason committed
5238 5239
		goto no_delete;
	}
Al Viro's avatar
Al Viro committed
5240
	/* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5241 5242
	if (!special_file(inode->i_mode))
		btrfs_wait_ordered_range(inode, 0, (u64)-1);
5243

5244 5245
	btrfs_free_io_failure_record(inode, 0, (u64)-1);

5246
	if (test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) {
5247
		BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5248
				 &BTRFS_I(inode)->runtime_flags));
5249 5250 5251
		goto no_delete;
	}

5252
	if (inode->i_nlink > 0) {
5253 5254
		BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
		       root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5255 5256 5257
		goto no_delete;
	}

5258 5259 5260 5261 5262 5263
	ret = btrfs_commit_inode_delayed_inode(inode);
	if (ret) {
		btrfs_orphan_del(NULL, inode);
		goto no_delete;
	}

5264
	rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
5265 5266 5267 5268
	if (!rsv) {
		btrfs_orphan_del(NULL, inode);
		goto no_delete;
	}
5269
	rsv->size = min_size;
5270
	rsv->failfast = 1;
5271
	global_rsv = &root->fs_info->global_block_rsv;
5272

5273
	btrfs_i_size_write(inode, 0);
5274

5275
	/*
5276 5277 5278 5279
	 * This is a bit simpler than btrfs_truncate since we've already
	 * reserved our space for our orphan item in the unlink, so we just
	 * need to reserve some slack space in case we add bytes and update
	 * inode item when doing the truncate.
5280
	 */
5281
	while (1) {
5282 5283
		ret = btrfs_block_rsv_refill(root, rsv, min_size,
					     BTRFS_RESERVE_FLUSH_LIMIT);
5284 5285 5286 5287 5288 5289 5290

		/*
		 * Try and steal from the global reserve since we will
		 * likely not use this space anyway, we want to try as
		 * hard as possible to get this to work.
		 */
		if (ret)
5291 5292 5293 5294
			steal_from_global++;
		else
			steal_from_global = 0;
		ret = 0;
5295

5296 5297 5298 5299 5300 5301 5302 5303 5304
		/*
		 * steal_from_global == 0: we reserved stuff, hooray!
		 * steal_from_global == 1: we didn't reserve stuff, boo!
		 * steal_from_global == 2: we've committed, still not a lot of
		 * room but maybe we'll have room in the global reserve this
		 * time.
		 * steal_from_global == 3: abandon all hope!
		 */
		if (steal_from_global > 2) {
5305 5306 5307
			btrfs_warn(root->fs_info,
				"Could not get space for a delete, will truncate on mount %d",
				ret);
5308 5309 5310
			btrfs_orphan_del(NULL, inode);
			btrfs_free_block_rsv(root, rsv);
			goto no_delete;
5311
		}
5312

5313
		trans = btrfs_join_transaction(root);
5314 5315 5316 5317
		if (IS_ERR(trans)) {
			btrfs_orphan_del(NULL, inode);
			btrfs_free_block_rsv(root, rsv);
			goto no_delete;
5318
		}
5319

5320
		/*
5321
		 * We can't just steal from the global reserve, we need to make
5322 5323 5324 5325 5326 5327
		 * sure there is room to do it, if not we need to commit and try
		 * again.
		 */
		if (steal_from_global) {
			if (!btrfs_check_space_for_delayed_refs(trans, root))
				ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5328
							      min_size, 0);
5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349
			else
				ret = -ENOSPC;
		}

		/*
		 * Couldn't steal from the global reserve, we have too much
		 * pending stuff built up, commit the transaction and try it
		 * again.
		 */
		if (ret) {
			ret = btrfs_commit_transaction(trans, root);
			if (ret) {
				btrfs_orphan_del(NULL, inode);
				btrfs_free_block_rsv(root, rsv);
				goto no_delete;
			}
			continue;
		} else {
			steal_from_global = 0;
		}

5350 5351
		trans->block_rsv = rsv;

5352
		ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5353
		if (ret != -ENOSPC && ret != -EAGAIN)
5354
			break;
5355

5356
		trans->block_rsv = &root->fs_info->trans_block_rsv;
5357 5358
		btrfs_end_transaction(trans, root);
		trans = NULL;
5359
		btrfs_btree_balance_dirty(root);
5360
	}
5361

5362 5363
	btrfs_free_block_rsv(root, rsv);

5364 5365 5366 5367
	/*
	 * Errors here aren't a big deal, it just means we leave orphan items
	 * in the tree.  They will be cleaned up on the next mount.
	 */
5368
	if (ret == 0) {
5369
		trans->block_rsv = root->orphan_block_rsv;
5370 5371 5372
		btrfs_orphan_del(trans, inode);
	} else {
		btrfs_orphan_del(NULL, inode);
5373
	}
5374

5375
	trans->block_rsv = &root->fs_info->trans_block_rsv;
5376 5377
	if (!(root == root->fs_info->tree_root ||
	      root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5378
		btrfs_return_ino(root, btrfs_ino(inode));
5379

5380
	btrfs_end_transaction(trans, root);
5381
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
5382
no_delete:
5383
	btrfs_remove_delayed_node(inode);
5384
	clear_inode(inode);
Chris Mason's avatar
Chris Mason committed
5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398
}

/*
 * this returns the key found in the dir entry in the location pointer.
 * If no dir entries were found, location->objectid is 0.
 */
static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
			       struct btrfs_key *location)
{
	const char *name = dentry->d_name.name;
	int namelen = dentry->d_name.len;
	struct btrfs_dir_item *di;
	struct btrfs_path *path;
	struct btrfs_root *root = BTRFS_I(dir)->root;
5399
	int ret = 0;
Chris Mason's avatar
Chris Mason committed
5400 5401

	path = btrfs_alloc_path();
5402 5403
	if (!path)
		return -ENOMEM;
5404

5405
	di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), name,
Chris Mason's avatar
Chris Mason committed
5406
				    namelen, 0);
5407 5408
	if (IS_ERR(di))
		ret = PTR_ERR(di);
5409

5410
	if (IS_ERR_OR_NULL(di))
5411
		goto out_err;
5412

5413
	btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
Chris Mason's avatar
Chris Mason committed
5414 5415 5416
out:
	btrfs_free_path(path);
	return ret;
5417 5418 5419
out_err:
	location->objectid = 0;
	goto out;
Chris Mason's avatar
Chris Mason committed
5420 5421 5422 5423 5424 5425 5426 5427
}

/*
 * when we hit a tree root in a directory, the btrfs part of the inode
 * needs to be changed to reflect the root directory of the tree root.  This
 * is kind of like crossing a mount point.
 */
static int fixup_tree_root_location(struct btrfs_root *root,
5428 5429 5430 5431
				    struct inode *dir,
				    struct dentry *dentry,
				    struct btrfs_key *location,
				    struct btrfs_root **sub_root)
Chris Mason's avatar
Chris Mason committed
5432
{
5433 5434 5435 5436
	struct btrfs_path *path;
	struct btrfs_root *new_root;
	struct btrfs_root_ref *ref;
	struct extent_buffer *leaf;
5437
	struct btrfs_key key;
5438 5439
	int ret;
	int err = 0;
Chris Mason's avatar
Chris Mason committed
5440

5441 5442 5443 5444 5445
	path = btrfs_alloc_path();
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
Chris Mason's avatar
Chris Mason committed
5446

5447
	err = -ENOENT;
5448 5449 5450 5451 5452 5453
	key.objectid = BTRFS_I(dir)->root->root_key.objectid;
	key.type = BTRFS_ROOT_REF_KEY;
	key.offset = location->objectid;

	ret = btrfs_search_slot(NULL, root->fs_info->tree_root, &key, path,
				0, 0);
5454 5455 5456 5457 5458
	if (ret) {
		if (ret < 0)
			err = ret;
		goto out;
	}
Chris Mason's avatar
Chris Mason committed
5459

5460 5461
	leaf = path->nodes[0];
	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5462
	if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) ||
5463 5464
	    btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
		goto out;
Chris Mason's avatar
Chris Mason committed
5465

5466 5467 5468 5469 5470 5471
	ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
				   (unsigned long)(ref + 1),
				   dentry->d_name.len);
	if (ret)
		goto out;

5472
	btrfs_release_path(path);
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487

	new_root = btrfs_read_fs_root_no_name(root->fs_info, location);
	if (IS_ERR(new_root)) {
		err = PTR_ERR(new_root);
		goto out;
	}

	*sub_root = new_root;
	location->objectid = btrfs_root_dirid(&new_root->root_item);
	location->type = BTRFS_INODE_ITEM_KEY;
	location->offset = 0;
	err = 0;
out:
	btrfs_free_path(path);
	return err;
Chris Mason's avatar
Chris Mason committed
5488 5489
}

5490 5491 5492 5493
static void inode_tree_add(struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_inode *entry;
5494 5495
	struct rb_node **p;
	struct rb_node *parent;
5496
	struct rb_node *new = &BTRFS_I(inode)->rb_node;
5497
	u64 ino = btrfs_ino(inode);
5498

Al Viro's avatar
Al Viro committed
5499
	if (inode_unhashed(inode))
5500
		return;
5501
	parent = NULL;
5502
	spin_lock(&root->inode_lock);
5503
	p = &root->inode_tree.rb_node;
5504 5505 5506 5507
	while (*p) {
		parent = *p;
		entry = rb_entry(parent, struct btrfs_inode, rb_node);

5508
		if (ino < btrfs_ino(&entry->vfs_inode))
5509
			p = &parent->rb_left;
5510
		else if (ino > btrfs_ino(&entry->vfs_inode))
5511
			p = &parent->rb_right;
5512 5513
		else {
			WARN_ON(!(entry->vfs_inode.i_state &
5514
				  (I_WILL_FREE | I_FREEING)));
5515
			rb_replace_node(parent, new, &root->inode_tree);
5516 5517
			RB_CLEAR_NODE(parent);
			spin_unlock(&root->inode_lock);
5518
			return;
5519 5520
		}
	}
5521 5522
	rb_link_node(new, parent, p);
	rb_insert_color(new, &root->inode_tree);
5523 5524 5525 5526 5527 5528
	spin_unlock(&root->inode_lock);
}

static void inode_tree_del(struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
5529
	int empty = 0;
5530

5531
	spin_lock(&root->inode_lock);
5532 5533 5534
	if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
		rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
		RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5535
		empty = RB_EMPTY_ROOT(&root->inode_tree);
5536
	}
5537
	spin_unlock(&root->inode_lock);
5538

5539
	if (empty && btrfs_root_refs(&root->root_item) == 0) {
5540 5541 5542 5543 5544 5545 5546 5547 5548
		synchronize_srcu(&root->fs_info->subvol_srcu);
		spin_lock(&root->inode_lock);
		empty = RB_EMPTY_ROOT(&root->inode_tree);
		spin_unlock(&root->inode_lock);
		if (empty)
			btrfs_add_dead_root(root);
	}
}

5549
void btrfs_invalidate_inodes(struct btrfs_root *root)
5550 5551 5552 5553 5554 5555 5556
{
	struct rb_node *node;
	struct rb_node *prev;
	struct btrfs_inode *entry;
	struct inode *inode;
	u64 objectid = 0;

5557 5558
	if (!test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
		WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5559 5560 5561 5562 5563 5564 5565 5566 5567

	spin_lock(&root->inode_lock);
again:
	node = root->inode_tree.rb_node;
	prev = NULL;
	while (node) {
		prev = node;
		entry = rb_entry(node, struct btrfs_inode, rb_node);

5568
		if (objectid < btrfs_ino(&entry->vfs_inode))
5569
			node = node->rb_left;
5570
		else if (objectid > btrfs_ino(&entry->vfs_inode))
5571 5572 5573 5574 5575 5576 5577
			node = node->rb_right;
		else
			break;
	}
	if (!node) {
		while (prev) {
			entry = rb_entry(prev, struct btrfs_inode, rb_node);
5578
			if (objectid <= btrfs_ino(&entry->vfs_inode)) {
5579 5580 5581 5582 5583 5584 5585 5586
				node = prev;
				break;
			}
			prev = rb_next(prev);
		}
	}
	while (node) {
		entry = rb_entry(node, struct btrfs_inode, rb_node);
5587
		objectid = btrfs_ino(&entry->vfs_inode) + 1;
5588 5589 5590 5591 5592 5593
		inode = igrab(&entry->vfs_inode);
		if (inode) {
			spin_unlock(&root->inode_lock);
			if (atomic_read(&inode->i_count) > 1)
				d_prune_aliases(inode);
			/*
5594
			 * btrfs_drop_inode will have it removed from
5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609
			 * the inode cache when its usage count
			 * hits zero.
			 */
			iput(inode);
			cond_resched();
			spin_lock(&root->inode_lock);
			goto again;
		}

		if (cond_resched_lock(&root->inode_lock))
			goto again;

		node = rb_next(node);
	}
	spin_unlock(&root->inode_lock);
5610 5611
}

5612 5613 5614
static int btrfs_init_locked_inode(struct inode *inode, void *p)
{
	struct btrfs_iget_args *args = p;
5615 5616 5617
	inode->i_ino = args->location->objectid;
	memcpy(&BTRFS_I(inode)->location, args->location,
	       sizeof(*args->location));
5618
	BTRFS_I(inode)->root = args->root;
Chris Mason's avatar
Chris Mason committed
5619 5620 5621 5622 5623 5624
	return 0;
}

static int btrfs_find_actor(struct inode *inode, void *opaque)
{
	struct btrfs_iget_args *args = opaque;
5625
	return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5626
		args->root == BTRFS_I(inode)->root;
Chris Mason's avatar
Chris Mason committed
5627 5628
}

5629
static struct inode *btrfs_iget_locked(struct super_block *s,
5630
				       struct btrfs_key *location,
5631
				       struct btrfs_root *root)
Chris Mason's avatar
Chris Mason committed
5632 5633 5634
{
	struct inode *inode;
	struct btrfs_iget_args args;
5635
	unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5636

5637
	args.location = location;
Chris Mason's avatar
Chris Mason committed
5638 5639
	args.root = root;

5640
	inode = iget5_locked(s, hashval, btrfs_find_actor,
Chris Mason's avatar
Chris Mason committed
5641 5642 5643 5644 5645
			     btrfs_init_locked_inode,
			     (void *)&args);
	return inode;
}

Balaji Rao's avatar
Balaji Rao committed
5646 5647 5648 5649
/* Get an inode object given its location and corresponding root.
 * Returns in *is_new if the inode was read from disk
 */
struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5650
			 struct btrfs_root *root, int *new)
Balaji Rao's avatar
Balaji Rao committed
5651 5652 5653
{
	struct inode *inode;

5654
	inode = btrfs_iget_locked(s, location, root);
Balaji Rao's avatar
Balaji Rao committed
5655
	if (!inode)
5656
		return ERR_PTR(-ENOMEM);
Balaji Rao's avatar
Balaji Rao committed
5657 5658

	if (inode->i_state & I_NEW) {
5659 5660 5661
		int ret;

		ret = btrfs_read_locked_inode(inode);
5662 5663 5664 5665 5666 5667
		if (!is_bad_inode(inode)) {
			inode_tree_add(inode);
			unlock_new_inode(inode);
			if (new)
				*new = 1;
		} else {
5668 5669
			unlock_new_inode(inode);
			iput(inode);
5670 5671
			ASSERT(ret < 0);
			inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5672 5673 5674
		}
	}

Balaji Rao's avatar
Balaji Rao committed
5675 5676 5677
	return inode;
}

5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688
static struct inode *new_simple_dir(struct super_block *s,
				    struct btrfs_key *key,
				    struct btrfs_root *root)
{
	struct inode *inode = new_inode(s);

	if (!inode)
		return ERR_PTR(-ENOMEM);

	BTRFS_I(inode)->root = root;
	memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5689
	set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5690 5691

	inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
Li Zefan's avatar
Li Zefan committed
5692
	inode->i_op = &btrfs_dir_ro_inode_operations;
5693
	inode->i_opflags &= ~IOP_XATTR;
5694 5695
	inode->i_fop = &simple_dir_operations;
	inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5696
	inode->i_mtime = current_time(inode);
5697 5698 5699
	inode->i_atime = inode->i_mtime;
	inode->i_ctime = inode->i_mtime;
	BTRFS_I(inode)->i_otime = inode->i_mtime;
5700 5701 5702 5703

	return inode;
}

5704
struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
Chris Mason's avatar
Chris Mason committed
5705
{
5706
	struct inode *inode;
5707
	struct btrfs_root *root = BTRFS_I(dir)->root;
Chris Mason's avatar
Chris Mason committed
5708 5709
	struct btrfs_root *sub_root = root;
	struct btrfs_key location;
5710
	int index;
5711
	int ret = 0;
Chris Mason's avatar
Chris Mason committed
5712 5713 5714

	if (dentry->d_name.len > BTRFS_NAME_LEN)
		return ERR_PTR(-ENAMETOOLONG);
5715

Jeff Layton's avatar
Jeff Layton committed
5716
	ret = btrfs_inode_by_name(dir, dentry, &location);
Chris Mason's avatar
Chris Mason committed
5717 5718
	if (ret < 0)
		return ERR_PTR(ret);
5719

5720
	if (location.objectid == 0)
5721
		return ERR_PTR(-ENOENT);
5722 5723

	if (location.type == BTRFS_INODE_ITEM_KEY) {
5724
		inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5725 5726 5727 5728 5729
		return inode;
	}

	BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY);

5730
	index = srcu_read_lock(&root->fs_info->subvol_srcu);
5731 5732 5733 5734 5735 5736 5737 5738
	ret = fixup_tree_root_location(root, dir, dentry,
				       &location, &sub_root);
	if (ret < 0) {
		if (ret != -ENOENT)
			inode = ERR_PTR(ret);
		else
			inode = new_simple_dir(dir->i_sb, &location, sub_root);
	} else {
5739
		inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
Chris Mason's avatar
Chris Mason committed
5740
	}
5741 5742
	srcu_read_unlock(&root->fs_info->subvol_srcu, index);

5743
	if (!IS_ERR(inode) && root != sub_root) {
5744 5745
		down_read(&root->fs_info->cleanup_work_sem);
		if (!(inode->i_sb->s_flags & MS_RDONLY))
5746
			ret = btrfs_orphan_cleanup(sub_root);
5747
		up_read(&root->fs_info->cleanup_work_sem);
5748 5749
		if (ret) {
			iput(inode);
5750
			inode = ERR_PTR(ret);
5751
		}
5752 5753
	}

5754 5755 5756
	return inode;
}

Nick Piggin's avatar
Nick Piggin committed
5757
static int btrfs_dentry_delete(const struct dentry *dentry)
5758 5759
{
	struct btrfs_root *root;
5760
	struct inode *inode = d_inode(dentry);
5761

Li Zefan's avatar
Li Zefan committed
5762
	if (!inode && !IS_ROOT(dentry))
5763
		inode = d_inode(dentry->d_parent);
5764

Li Zefan's avatar
Li Zefan committed
5765 5766
	if (inode) {
		root = BTRFS_I(inode)->root;
5767 5768
		if (btrfs_root_refs(&root->root_item) == 0)
			return 1;
Li Zefan's avatar
Li Zefan committed
5769 5770 5771

		if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
			return 1;
5772
	}
5773 5774 5775
	return 0;
}

5776 5777
static void btrfs_dentry_release(struct dentry *dentry)
{
5778
	kfree(dentry->d_fsdata);
5779 5780
}

5781
static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5782
				   unsigned int flags)
5783
{
5784
	struct inode *inode;
5785

5786 5787 5788 5789 5790 5791 5792 5793
	inode = btrfs_lookup_dentry(dir, dentry);
	if (IS_ERR(inode)) {
		if (PTR_ERR(inode) == -ENOENT)
			inode = NULL;
		else
			return ERR_CAST(inode);
	}

5794
	return d_splice_alias(inode, dentry);
Chris Mason's avatar
Chris Mason committed
5795 5796
}

5797
unsigned char btrfs_filetype_table[] = {
Chris Mason's avatar
Chris Mason committed
5798 5799 5800
	DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
};

Al Viro's avatar
Al Viro committed
5801
static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
Chris Mason's avatar
Chris Mason committed
5802
{
Al Viro's avatar
Al Viro committed
5803
	struct inode *inode = file_inode(file);
Chris Mason's avatar
Chris Mason committed
5804 5805 5806 5807
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_item *item;
	struct btrfs_dir_item *di;
	struct btrfs_key key;
5808
	struct btrfs_key found_key;
Chris Mason's avatar
Chris Mason committed
5809
	struct btrfs_path *path;
5810 5811
	struct list_head ins_list;
	struct list_head del_list;
Chris Mason's avatar
Chris Mason committed
5812
	int ret;
5813
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
5814 5815 5816 5817 5818 5819 5820
	int slot;
	unsigned char d_type;
	int over = 0;
	u32 di_cur;
	u32 di_total;
	u32 di_len;
	int key_type = BTRFS_DIR_INDEX_KEY;
5821 5822 5823
	char tmp_name[32];
	char *name_ptr;
	int name_len;
Al Viro's avatar
Al Viro committed
5824
	int is_curr = 0;	/* ctx->pos points to the current index? */
5825
	bool emitted;
5826
	bool put = false;
Chris Mason's avatar
Chris Mason committed
5827 5828 5829 5830

	/* FIXME, use a real flag for deciding about the key type */
	if (root->fs_info->tree_root == root)
		key_type = BTRFS_DIR_ITEM_KEY;
5831

Al Viro's avatar
Al Viro committed
5832 5833 5834
	if (!dir_emit_dots(file, ctx))
		return 0;

5835
	path = btrfs_alloc_path();
5836 5837
	if (!path)
		return -ENOMEM;
Chris Mason's avatar
Chris Mason committed
5838

5839
	path->reada = READA_FORWARD;
5840

5841 5842 5843
	if (key_type == BTRFS_DIR_INDEX_KEY) {
		INIT_LIST_HEAD(&ins_list);
		INIT_LIST_HEAD(&del_list);
5844 5845
		put = btrfs_readdir_get_delayed_items(inode, &ins_list,
						      &del_list);
5846 5847
	}

5848
	key.type = key_type;
Al Viro's avatar
Al Viro committed
5849
	key.offset = ctx->pos;
5850
	key.objectid = btrfs_ino(inode);
5851

Chris Mason's avatar
Chris Mason committed
5852 5853 5854
	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0)
		goto err;
5855

5856
	emitted = false;
5857
	while (1) {
5858
		leaf = path->nodes[0];
Chris Mason's avatar
Chris Mason committed
5859
		slot = path->slots[0];
5860 5861 5862 5863 5864 5865 5866
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				goto err;
			else if (ret > 0)
				break;
			continue;
Chris Mason's avatar
Chris Mason committed
5867
		}
5868

5869
		item = btrfs_item_nr(slot);
5870 5871 5872
		btrfs_item_key_to_cpu(leaf, &found_key, slot);

		if (found_key.objectid != key.objectid)
Chris Mason's avatar
Chris Mason committed
5873
			break;
5874
		if (found_key.type != key_type)
Chris Mason's avatar
Chris Mason committed
5875
			break;
Al Viro's avatar
Al Viro committed
5876
		if (found_key.offset < ctx->pos)
5877
			goto next;
5878 5879 5880 5881
		if (key_type == BTRFS_DIR_INDEX_KEY &&
		    btrfs_should_delete_dir_index(&del_list,
						  found_key.offset))
			goto next;
5882

Al Viro's avatar
Al Viro committed
5883
		ctx->pos = found_key.offset;
5884
		is_curr = 1;
5885

Chris Mason's avatar
Chris Mason committed
5886 5887
		di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
		di_cur = 0;
5888
		di_total = btrfs_item_size(leaf, item);
5889 5890

		while (di_cur < di_total) {
5891 5892
			struct btrfs_key location;

5893 5894 5895
			if (verify_dir_item(root, leaf, di))
				break;

5896
			name_len = btrfs_dir_name_len(leaf, di);
5897
			if (name_len <= sizeof(tmp_name)) {
5898 5899
				name_ptr = tmp_name;
			} else {
5900
				name_ptr = kmalloc(name_len, GFP_KERNEL);
5901 5902 5903 5904
				if (!name_ptr) {
					ret = -ENOMEM;
					goto err;
				}
5905 5906 5907 5908 5909 5910
			}
			read_extent_buffer(leaf, name_ptr,
					   (unsigned long)(di + 1), name_len);

			d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
			btrfs_dir_item_key_to_cpu(leaf, di, &location);
5911

5912

5913
			/* is this a reference to our own snapshot? If so
Arne Jansen's avatar
Arne Jansen committed
5914 5915 5916 5917 5918 5919 5920
			 * skip it.
			 *
			 * In contrast to old kernels, we insert the snapshot's
			 * dir item and dir index after it has been created, so
			 * we won't find a reference to our own snapshot. We
			 * still keep the following code for backward
			 * compatibility.
5921 5922 5923 5924 5925 5926
			 */
			if (location.type == BTRFS_ROOT_ITEM_KEY &&
			    location.objectid == root->root_key.objectid) {
				over = 0;
				goto skip;
			}
Al Viro's avatar
Al Viro committed
5927 5928
			over = !dir_emit(ctx, name_ptr, name_len,
				       location.objectid, d_type);
5929

5930
skip:
5931 5932 5933
			if (name_ptr != tmp_name)
				kfree(name_ptr);

Chris Mason's avatar
Chris Mason committed
5934 5935
			if (over)
				goto nopos;
5936
			emitted = true;
Josef Bacik's avatar
Josef Bacik committed
5937
			di_len = btrfs_dir_name_len(leaf, di) +
5938
				 btrfs_dir_data_len(leaf, di) + sizeof(*di);
Chris Mason's avatar
Chris Mason committed
5939 5940 5941
			di_cur += di_len;
			di = (struct btrfs_dir_item *)((char *)di + di_len);
		}
5942 5943
next:
		path->slots[0]++;
Chris Mason's avatar
Chris Mason committed
5944
	}
5945

5946 5947
	if (key_type == BTRFS_DIR_INDEX_KEY) {
		if (is_curr)
Al Viro's avatar
Al Viro committed
5948
			ctx->pos++;
5949
		ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list, &emitted);
5950 5951 5952 5953
		if (ret)
			goto nopos;
	}

5954 5955 5956 5957 5958 5959 5960 5961 5962
	/*
	 * If we haven't emitted any dir entry, we must not touch ctx->pos as
	 * it was was set to the termination value in previous call. We assume
	 * that "." and ".." were emitted if we reach this point and set the
	 * termination value as well for an empty directory.
	 */
	if (ctx->pos > 2 && !emitted)
		goto nopos;

5963
	/* Reached end of directory/root. Bump pos past the last item. */
5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988
	ctx->pos++;

	/*
	 * Stop new entries from being returned after we return the last
	 * entry.
	 *
	 * New directory entries are assigned a strictly increasing
	 * offset.  This means that new entries created during readdir
	 * are *guaranteed* to be seen in the future by that readdir.
	 * This has broken buggy programs which operate on names as
	 * they're returned by readdir.  Until we re-use freed offsets
	 * we have this hack to stop new entries from being returned
	 * under the assumption that they'll never reach this huge
	 * offset.
	 *
	 * This is being careful not to overflow 32bit loff_t unless the
	 * last entry requires it because doing so has broken 32bit apps
	 * in the past.
	 */
	if (key_type == BTRFS_DIR_INDEX_KEY) {
		if (ctx->pos >= INT_MAX)
			ctx->pos = LLONG_MAX;
		else
			ctx->pos = INT_MAX;
	}
Chris Mason's avatar
Chris Mason committed
5989 5990 5991
nopos:
	ret = 0;
err:
5992 5993
	if (put)
		btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
Chris Mason's avatar
Chris Mason committed
5994 5995 5996 5997
	btrfs_free_path(path);
	return ret;
}

5998
int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
Chris Mason's avatar
Chris Mason committed
5999 6000 6001 6002
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_trans_handle *trans;
	int ret = 0;
6003
	bool nolock = false;
Chris Mason's avatar
Chris Mason committed
6004

6005
	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6006 6007
		return 0;

6008
	if (btrfs_fs_closing(root->fs_info) && btrfs_is_free_space_inode(inode))
6009
		nolock = true;
6010

6011
	if (wbc->sync_mode == WB_SYNC_ALL) {
6012
		if (nolock)
6013
			trans = btrfs_join_transaction_nolock(root);
6014
		else
6015
			trans = btrfs_join_transaction(root);
6016 6017
		if (IS_ERR(trans))
			return PTR_ERR(trans);
6018
		ret = btrfs_commit_transaction(trans, root);
Chris Mason's avatar
Chris Mason committed
6019 6020 6021 6022 6023
	}
	return ret;
}

/*
6024
 * This is somewhat expensive, updating the tree every time the
Chris Mason's avatar
Chris Mason committed
6025 6026 6027 6028
 * inode changes.  But, it is most likely to find the inode in cache.
 * FIXME, needs more benchmarking...there are no reasons other than performance
 * to keep or drop this code.
 */
6029
static int btrfs_dirty_inode(struct inode *inode)
Chris Mason's avatar
Chris Mason committed
6030 6031 6032
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_trans_handle *trans;
6033 6034
	int ret;

6035
	if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6036
		return 0;
Chris Mason's avatar
Chris Mason committed
6037

6038
	trans = btrfs_join_transaction(root);
6039 6040
	if (IS_ERR(trans))
		return PTR_ERR(trans);
6041 6042

	ret = btrfs_update_inode(trans, root, inode);
6043 6044 6045 6046
	if (ret && ret == -ENOSPC) {
		/* whoops, lets try again with the full transaction */
		btrfs_end_transaction(trans, root);
		trans = btrfs_start_transaction(root, 1);
6047 6048
		if (IS_ERR(trans))
			return PTR_ERR(trans);
6049

6050 6051
		ret = btrfs_update_inode(trans, root, inode);
	}
Chris Mason's avatar
Chris Mason committed
6052
	btrfs_end_transaction(trans, root);
6053 6054
	if (BTRFS_I(inode)->delayed_node)
		btrfs_balance_delayed_items(root);
6055 6056 6057 6058 6059 6060 6061 6062

	return ret;
}

/*
 * This is a copy of file_update_time.  We need this so we can return error on
 * ENOSPC for updating the inode in the case of file write and mmap writes.
 */
6063 6064
static int btrfs_update_time(struct inode *inode, struct timespec *now,
			     int flags)
6065
{
6066 6067 6068 6069 6070
	struct btrfs_root *root = BTRFS_I(inode)->root;

	if (btrfs_root_readonly(root))
		return -EROFS;

6071
	if (flags & S_VERSION)
6072
		inode_inc_iversion(inode);
6073 6074 6075 6076 6077 6078 6079
	if (flags & S_CTIME)
		inode->i_ctime = *now;
	if (flags & S_MTIME)
		inode->i_mtime = *now;
	if (flags & S_ATIME)
		inode->i_atime = *now;
	return btrfs_dirty_inode(inode);
Chris Mason's avatar
Chris Mason committed
6080 6081
}

6082 6083 6084 6085 6086
/*
 * find the highest existing sequence number in a directory
 * and then set the in-memory index_cnt variable to reflect
 * free sequence numbers
 */
6087 6088 6089 6090 6091 6092 6093 6094
static int btrfs_set_inode_index_count(struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_key key, found_key;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	int ret;

6095
	key.objectid = btrfs_ino(inode);
6096
	key.type = BTRFS_DIR_INDEX_KEY;
6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126
	key.offset = (u64)-1;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
	/* FIXME: we should be able to handle this */
	if (ret == 0)
		goto out;
	ret = 0;

	/*
	 * MAGIC NUMBER EXPLANATION:
	 * since we search a directory based on f_pos we have to start at 2
	 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
	 * else has to start at 2
	 */
	if (path->slots[0] == 0) {
		BTRFS_I(inode)->index_cnt = 2;
		goto out;
	}

	path->slots[0]--;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);

6127
	if (found_key.objectid != btrfs_ino(inode) ||
6128
	    found_key.type != BTRFS_DIR_INDEX_KEY) {
6129 6130 6131 6132 6133 6134 6135 6136 6137 6138
		BTRFS_I(inode)->index_cnt = 2;
		goto out;
	}

	BTRFS_I(inode)->index_cnt = found_key.offset + 1;
out:
	btrfs_free_path(path);
	return ret;
}

6139 6140 6141 6142
/*
 * helper to find a free sequence number in a given directory.  This current
 * code is very simple, later versions will do smarter things in the btree
 */
6143
int btrfs_set_inode_index(struct inode *dir, u64 *index)
6144 6145 6146 6147
{
	int ret = 0;

	if (BTRFS_I(dir)->index_cnt == (u64)-1) {
6148 6149 6150 6151 6152 6153
		ret = btrfs_inode_delayed_dir_index_count(dir);
		if (ret) {
			ret = btrfs_set_inode_index_count(dir);
			if (ret)
				return ret;
		}
6154 6155
	}

6156
	*index = BTRFS_I(dir)->index_cnt;
6157 6158 6159 6160 6161
	BTRFS_I(dir)->index_cnt++;

	return ret;
}

6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172
static int btrfs_insert_inode_locked(struct inode *inode)
{
	struct btrfs_iget_args args;
	args.location = &BTRFS_I(inode)->location;
	args.root = BTRFS_I(inode)->root;

	return insert_inode_locked4(inode,
		   btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
		   btrfs_find_actor, &args);
}

Chris Mason's avatar
Chris Mason committed
6173 6174
static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
6175
				     struct inode *dir,
6176
				     const char *name, int name_len,
Al Viro's avatar
Al Viro committed
6177 6178
				     u64 ref_objectid, u64 objectid,
				     umode_t mode, u64 *index)
Chris Mason's avatar
Chris Mason committed
6179 6180
{
	struct inode *inode;
6181
	struct btrfs_inode_item *inode_item;
Chris Mason's avatar
Chris Mason committed
6182
	struct btrfs_key *location;
6183
	struct btrfs_path *path;
6184 6185 6186
	struct btrfs_inode_ref *ref;
	struct btrfs_key key[2];
	u32 sizes[2];
6187
	int nitems = name ? 2 : 1;
6188
	unsigned long ptr;
Chris Mason's avatar
Chris Mason committed
6189 6190
	int ret;

6191
	path = btrfs_alloc_path();
6192 6193
	if (!path)
		return ERR_PTR(-ENOMEM);
6194

Chris Mason's avatar
Chris Mason committed
6195
	inode = new_inode(root->fs_info->sb);
6196 6197
	if (!inode) {
		btrfs_free_path(path);
Chris Mason's avatar
Chris Mason committed
6198
		return ERR_PTR(-ENOMEM);
6199
	}
Chris Mason's avatar
Chris Mason committed
6200

6201 6202 6203 6204 6205 6206 6207
	/*
	 * O_TMPFILE, set link count to 0, so that after this point,
	 * we fill in an inode item with the correct link count.
	 */
	if (!name)
		set_nlink(inode, 0);

6208 6209 6210 6211 6212 6213
	/*
	 * we have to initialize this early, so we can reclaim the inode
	 * number if we fail afterwards in this function.
	 */
	inode->i_ino = objectid;

6214
	if (dir && name) {
6215 6216
		trace_btrfs_inode_request(dir);

6217
		ret = btrfs_set_inode_index(dir, index);
6218
		if (ret) {
6219
			btrfs_free_path(path);
6220
			iput(inode);
6221
			return ERR_PTR(ret);
6222
		}
6223 6224
	} else if (dir) {
		*index = 0;
6225 6226 6227 6228 6229 6230 6231
	}
	/*
	 * index_cnt is ignored for everything but a dir,
	 * btrfs_get_inode_index_count has an explanation for the magic
	 * number
	 */
	BTRFS_I(inode)->index_cnt = 2;
6232
	BTRFS_I(inode)->dir_index = *index;
Chris Mason's avatar
Chris Mason committed
6233
	BTRFS_I(inode)->root = root;
6234
	BTRFS_I(inode)->generation = trans->transid;
6235
	inode->i_generation = BTRFS_I(inode)->generation;
6236

Josef Bacik's avatar
Josef Bacik committed
6237 6238 6239 6240 6241 6242 6243 6244
	/*
	 * We could have gotten an inode number from somebody who was fsynced
	 * and then removed in this same transaction, so let's just set full
	 * sync since it will be a full sync anyway and this will blow away the
	 * old info in the log.
	 */
	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);

6245
	key[0].objectid = objectid;
6246
	key[0].type = BTRFS_INODE_ITEM_KEY;
6247 6248 6249
	key[0].offset = 0;

	sizes[0] = sizeof(struct btrfs_inode_item);
6250 6251 6252 6253 6254 6255 6256 6257 6258

	if (name) {
		/*
		 * Start new inodes with an inode_ref. This is slightly more
		 * efficient for small numbers of hard links since they will
		 * be packed into one item. Extended refs will kick in if we
		 * add more hard links than can fit in the ref item.
		 */
		key[1].objectid = objectid;
6259
		key[1].type = BTRFS_INODE_REF_KEY;
6260 6261 6262 6263
		key[1].offset = ref_objectid;

		sizes[1] = name_len + sizeof(*ref);
	}
6264

6265 6266 6267
	location = &BTRFS_I(inode)->location;
	location->objectid = objectid;
	location->offset = 0;
6268
	location->type = BTRFS_INODE_ITEM_KEY;
6269 6270 6271 6272 6273

	ret = btrfs_insert_inode_locked(inode);
	if (ret < 0)
		goto fail;

6274
	path->leave_spinning = 1;
6275
	ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6276
	if (ret != 0)
6277
		goto fail_unlock;
6278

6279
	inode_init_owner(inode, dir, mode);
6280
	inode_set_bytes(inode, 0);
6281

6282
	inode->i_mtime = current_time(inode);
6283 6284 6285 6286
	inode->i_atime = inode->i_mtime;
	inode->i_ctime = inode->i_mtime;
	BTRFS_I(inode)->i_otime = inode->i_mtime;

6287 6288
	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
				  struct btrfs_inode_item);
6289 6290
	memset_extent_buffer(path->nodes[0], 0, (unsigned long)inode_item,
			     sizeof(*inode_item));
6291
	fill_inode_item(trans, path->nodes[0], inode_item, inode);
6292

6293 6294 6295 6296 6297 6298 6299 6300
	if (name) {
		ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
				     struct btrfs_inode_ref);
		btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
		btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
		ptr = (unsigned long)(ref + 1);
		write_extent_buffer(path->nodes[0], name, ptr, name_len);
	}
6301

6302 6303 6304
	btrfs_mark_buffer_dirty(path->nodes[0]);
	btrfs_free_path(path);

6305 6306
	btrfs_inherit_iflags(inode, dir);

6307
	if (S_ISREG(mode)) {
6308
		if (btrfs_test_opt(root->fs_info, NODATASUM))
6309
			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6310
		if (btrfs_test_opt(root->fs_info, NODATACOW))
6311 6312
			BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
				BTRFS_INODE_NODATASUM;
6313 6314
	}

6315
	inode_tree_add(inode);
6316 6317

	trace_btrfs_inode_new(inode);
6318
	btrfs_set_inode_last_trans(trans, inode);
6319

6320 6321
	btrfs_update_root_times(trans, root);

6322 6323 6324 6325 6326 6327
	ret = btrfs_inode_inherit_props(trans, inode, dir);
	if (ret)
		btrfs_err(root->fs_info,
			  "error inheriting props for ino %llu (root %llu): %d",
			  btrfs_ino(inode), root->root_key.objectid, ret);

Chris Mason's avatar
Chris Mason committed
6328
	return inode;
6329 6330 6331

fail_unlock:
	unlock_new_inode(inode);
6332
fail:
6333
	if (dir && name)
6334
		BTRFS_I(dir)->index_cnt--;
6335
	btrfs_free_path(path);
6336
	iput(inode);
6337
	return ERR_PTR(ret);
Chris Mason's avatar
Chris Mason committed
6338 6339 6340 6341 6342 6343 6344
}

static inline u8 btrfs_inode_type(struct inode *inode)
{
	return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
}

6345 6346 6347 6348 6349 6350
/*
 * utility function to add 'inode' into 'parent_inode' with
 * a give name and a given sequence number.
 * if 'add_backref' is true, also insert a backref from the
 * inode to the parent directory.
 */
6351 6352 6353
int btrfs_add_link(struct btrfs_trans_handle *trans,
		   struct inode *parent_inode, struct inode *inode,
		   const char *name, int name_len, int add_backref, u64 index)
Chris Mason's avatar
Chris Mason committed
6354
{
6355
	int ret = 0;
Chris Mason's avatar
Chris Mason committed
6356
	struct btrfs_key key;
6357
	struct btrfs_root *root = BTRFS_I(parent_inode)->root;
6358 6359
	u64 ino = btrfs_ino(inode);
	u64 parent_ino = btrfs_ino(parent_inode);
6360

6361
	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6362 6363
		memcpy(&key, &BTRFS_I(inode)->root->root_key, sizeof(key));
	} else {
6364
		key.objectid = ino;
6365
		key.type = BTRFS_INODE_ITEM_KEY;
6366 6367 6368
		key.offset = 0;
	}

6369
	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6370 6371
		ret = btrfs_add_root_ref(trans, root->fs_info->tree_root,
					 key.objectid, root->root_key.objectid,
6372
					 parent_ino, index, name, name_len);
6373
	} else if (add_backref) {
6374 6375
		ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
					     parent_ino, index);
6376
	}
Chris Mason's avatar
Chris Mason committed
6377

6378 6379 6380
	/* Nothing to clean up yet */
	if (ret)
		return ret;
6381

6382 6383 6384
	ret = btrfs_insert_dir_item(trans, root, name, name_len,
				    parent_inode, &key,
				    btrfs_inode_type(inode), index);
6385
	if (ret == -EEXIST || ret == -EOVERFLOW)
6386 6387
		goto fail_dir_item;
	else if (ret) {
6388
		btrfs_abort_transaction(trans, ret);
6389
		return ret;
Chris Mason's avatar
Chris Mason committed
6390
	}
6391 6392 6393

	btrfs_i_size_write(parent_inode, parent_inode->i_size +
			   name_len * 2);
6394
	inode_inc_iversion(parent_inode);
6395
	parent_inode->i_mtime = parent_inode->i_ctime =
6396
		current_time(parent_inode);
6397 6398
	ret = btrfs_update_inode(trans, root, parent_inode);
	if (ret)
6399
		btrfs_abort_transaction(trans, ret);
Chris Mason's avatar
Chris Mason committed
6400
	return ret;
6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417

fail_dir_item:
	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
		u64 local_index;
		int err;
		err = btrfs_del_root_ref(trans, root->fs_info->tree_root,
				 key.objectid, root->root_key.objectid,
				 parent_ino, &local_index, name, name_len);

	} else if (add_backref) {
		u64 local_index;
		int err;

		err = btrfs_del_inode_ref(trans, root, name, name_len,
					  ino, parent_ino, &local_index);
	}
	return ret;
Chris Mason's avatar
Chris Mason committed
6418 6419 6420
}

static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6421 6422
			    struct inode *dir, struct dentry *dentry,
			    struct inode *inode, int backref, u64 index)
Chris Mason's avatar
Chris Mason committed
6423
{
6424 6425 6426
	int err = btrfs_add_link(trans, dir, inode,
				 dentry->d_name.name, dentry->d_name.len,
				 backref, index);
Chris Mason's avatar
Chris Mason committed
6427 6428 6429 6430 6431
	if (err > 0)
		err = -EEXIST;
	return err;
}

Josef Bacik's avatar
Josef Bacik committed
6432
static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
Al Viro's avatar
Al Viro committed
6433
			umode_t mode, dev_t rdev)
Josef Bacik's avatar
Josef Bacik committed
6434 6435 6436
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(dir)->root;
6437
	struct inode *inode = NULL;
Josef Bacik's avatar
Josef Bacik committed
6438 6439 6440
	int err;
	int drop_inode = 0;
	u64 objectid;
6441
	u64 index = 0;
Josef Bacik's avatar
Josef Bacik committed
6442

Josef Bacik's avatar
Josef Bacik committed
6443 6444 6445 6446 6447
	/*
	 * 2 for inode item and ref
	 * 2 for dir items
	 * 1 for xattr if selinux is on
	 */
6448 6449 6450
	trans = btrfs_start_transaction(root, 5);
	if (IS_ERR(trans))
		return PTR_ERR(trans);
6451

6452 6453 6454 6455
	err = btrfs_find_free_ino(root, &objectid);
	if (err)
		goto out_unlock;

6456
	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6457
				dentry->d_name.len, btrfs_ino(dir), objectid,
6458
				mode, &index);
6459 6460
	if (IS_ERR(inode)) {
		err = PTR_ERR(inode);
Josef Bacik's avatar
Josef Bacik committed
6461
		goto out_unlock;
6462
	}
Josef Bacik's avatar
Josef Bacik committed
6463

6464 6465 6466 6467 6468 6469 6470
	/*
	* If the active LSM wants to access the inode during
	* d_instantiate it needs these. Smack checks to see
	* if the filesystem supports xattrs by looking at the
	* ops vector.
	*/
	inode->i_op = &btrfs_special_inode_operations;
6471 6472 6473
	init_special_inode(inode, inode->i_mode, rdev);

	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
Josef Bacik's avatar
Josef Bacik committed
6474
	if (err)
6475 6476 6477 6478 6479 6480
		goto out_unlock_inode;

	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
	if (err) {
		goto out_unlock_inode;
	} else {
6481
		btrfs_update_inode(trans, root, inode);
6482
		unlock_new_inode(inode);
6483
		d_instantiate(dentry, inode);
Josef Bacik's avatar
Josef Bacik committed
6484
	}
6485

Josef Bacik's avatar
Josef Bacik committed
6486
out_unlock:
6487
	btrfs_end_transaction(trans, root);
6488
	btrfs_balance_delayed_items(root);
6489
	btrfs_btree_balance_dirty(root);
Josef Bacik's avatar
Josef Bacik committed
6490 6491 6492 6493 6494
	if (drop_inode) {
		inode_dec_link_count(inode);
		iput(inode);
	}
	return err;
6495 6496 6497 6498 6499 6500

out_unlock_inode:
	drop_inode = 1;
	unlock_new_inode(inode);
	goto out_unlock;

Josef Bacik's avatar
Josef Bacik committed
6501 6502
}

Chris Mason's avatar
Chris Mason committed
6503
static int btrfs_create(struct inode *dir, struct dentry *dentry,
Al Viro's avatar
Al Viro committed
6504
			umode_t mode, bool excl)
Chris Mason's avatar
Chris Mason committed
6505 6506 6507
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(dir)->root;
6508
	struct inode *inode = NULL;
6509
	int drop_inode_on_err = 0;
6510
	int err;
Chris Mason's avatar
Chris Mason committed
6511
	u64 objectid;
6512
	u64 index = 0;
Chris Mason's avatar
Chris Mason committed
6513

Josef Bacik's avatar
Josef Bacik committed
6514 6515 6516 6517 6518
	/*
	 * 2 for inode item and ref
	 * 2 for dir items
	 * 1 for xattr if selinux is on
	 */
6519 6520 6521
	trans = btrfs_start_transaction(root, 5);
	if (IS_ERR(trans))
		return PTR_ERR(trans);
Josef Bacik's avatar
Josef Bacik committed
6522

6523 6524 6525 6526
	err = btrfs_find_free_ino(root, &objectid);
	if (err)
		goto out_unlock;

6527
	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6528
				dentry->d_name.len, btrfs_ino(dir), objectid,
6529
				mode, &index);
6530 6531
	if (IS_ERR(inode)) {
		err = PTR_ERR(inode);
Chris Mason's avatar
Chris Mason committed
6532
		goto out_unlock;
6533
	}
6534
	drop_inode_on_err = 1;
6535 6536 6537 6538 6539 6540 6541 6542
	/*
	* If the active LSM wants to access the inode during
	* d_instantiate it needs these. Smack checks to see
	* if the filesystem supports xattrs by looking at the
	* ops vector.
	*/
	inode->i_fop = &btrfs_file_operations;
	inode->i_op = &btrfs_file_inode_operations;
6543 6544 6545 6546 6547 6548 6549 6550 6551
	inode->i_mapping->a_ops = &btrfs_aops;

	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
	if (err)
		goto out_unlock_inode;

	err = btrfs_update_inode(trans, root, inode);
	if (err)
		goto out_unlock_inode;
6552

6553
	err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
Chris Mason's avatar
Chris Mason committed
6554
	if (err)
6555
		goto out_unlock_inode;
6556 6557

	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6558
	unlock_new_inode(inode);
6559 6560
	d_instantiate(dentry, inode);

Chris Mason's avatar
Chris Mason committed
6561
out_unlock:
6562
	btrfs_end_transaction(trans, root);
6563
	if (err && drop_inode_on_err) {
Chris Mason's avatar
Chris Mason committed
6564 6565 6566
		inode_dec_link_count(inode);
		iput(inode);
	}
6567
	btrfs_balance_delayed_items(root);
6568
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
6569
	return err;
6570 6571 6572 6573 6574

out_unlock_inode:
	unlock_new_inode(inode);
	goto out_unlock;

Chris Mason's avatar
Chris Mason committed
6575 6576 6577 6578 6579
}

static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
		      struct dentry *dentry)
{
6580
	struct btrfs_trans_handle *trans = NULL;
Chris Mason's avatar
Chris Mason committed
6581
	struct btrfs_root *root = BTRFS_I(dir)->root;
6582
	struct inode *inode = d_inode(old_dentry);
6583
	u64 index;
Chris Mason's avatar
Chris Mason committed
6584 6585 6586
	int err;
	int drop_inode = 0;

6587 6588
	/* do not allow sys_link's with other subvols of the same device */
	if (root->objectid != BTRFS_I(inode)->root->objectid)
6589
		return -EXDEV;
6590

Mark Fasheh's avatar
Mark Fasheh committed
6591
	if (inode->i_nlink >= BTRFS_LINK_MAX)
6592
		return -EMLINK;
6593

6594
	err = btrfs_set_inode_index(dir, &index);
6595 6596 6597
	if (err)
		goto fail;

6598
	/*
6599
	 * 2 items for inode and inode ref
6600
	 * 2 items for dir items
6601
	 * 1 item for parent inode
6602
	 */
6603
	trans = btrfs_start_transaction(root, 5);
6604 6605
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
6606
		trans = NULL;
6607 6608
		goto fail;
	}
6609

6610 6611
	/* There are several dir indexes for this inode, clear the cache. */
	BTRFS_I(inode)->dir_index = 0ULL;
Zach Brown's avatar
Zach Brown committed
6612
	inc_nlink(inode);
6613
	inode_inc_iversion(inode);
6614
	inode->i_ctime = current_time(inode);
Al Viro's avatar
Al Viro committed
6615
	ihold(inode);
6616
	set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6617

6618
	err = btrfs_add_nondir(trans, dir, dentry, inode, 1, index);
6619

6620
	if (err) {
6621
		drop_inode = 1;
6622
	} else {
6623
		struct dentry *parent = dentry->d_parent;
6624
		err = btrfs_update_inode(trans, root, inode);
6625 6626
		if (err)
			goto fail;
6627 6628 6629 6630 6631 6632 6633 6634 6635
		if (inode->i_nlink == 1) {
			/*
			 * If new hard link count is 1, it's a file created
			 * with open(2) O_TMPFILE flag.
			 */
			err = btrfs_orphan_del(trans, inode);
			if (err)
				goto fail;
		}
6636
		d_instantiate(dentry, inode);
6637
		btrfs_log_new_name(trans, inode, NULL, parent);
6638
	}
Chris Mason's avatar
Chris Mason committed
6639

6640
	btrfs_balance_delayed_items(root);
6641
fail:
6642 6643
	if (trans)
		btrfs_end_transaction(trans, root);
Chris Mason's avatar
Chris Mason committed
6644 6645 6646 6647
	if (drop_inode) {
		inode_dec_link_count(inode);
		iput(inode);
	}
6648
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
6649 6650 6651
	return err;
}

6652
static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
Chris Mason's avatar
Chris Mason committed
6653
{
6654
	struct inode *inode = NULL;
Chris Mason's avatar
Chris Mason committed
6655 6656 6657 6658
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(dir)->root;
	int err = 0;
	int drop_on_err = 0;
6659
	u64 objectid = 0;
6660
	u64 index = 0;
Chris Mason's avatar
Chris Mason committed
6661

Josef Bacik's avatar
Josef Bacik committed
6662 6663 6664 6665 6666
	/*
	 * 2 items for inode and ref
	 * 2 items for dir items
	 * 1 for xattr if selinux is on
	 */
6667 6668 6669
	trans = btrfs_start_transaction(root, 5);
	if (IS_ERR(trans))
		return PTR_ERR(trans);
Chris Mason's avatar
Chris Mason committed
6670

6671 6672 6673 6674
	err = btrfs_find_free_ino(root, &objectid);
	if (err)
		goto out_fail;

6675
	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6676
				dentry->d_name.len, btrfs_ino(dir), objectid,
6677
				S_IFDIR | mode, &index);
Chris Mason's avatar
Chris Mason committed
6678 6679 6680 6681
	if (IS_ERR(inode)) {
		err = PTR_ERR(inode);
		goto out_fail;
	}
6682

Chris Mason's avatar
Chris Mason committed
6683
	drop_on_err = 1;
6684 6685 6686
	/* these must be set before we unlock the inode */
	inode->i_op = &btrfs_dir_inode_operations;
	inode->i_fop = &btrfs_dir_file_operations;
Josef Bacik's avatar
Josef Bacik committed
6687

6688
	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
Josef Bacik's avatar
Josef Bacik committed
6689
	if (err)
6690
		goto out_fail_inode;
Chris Mason's avatar
Chris Mason committed
6691

6692
	btrfs_i_size_write(inode, 0);
Chris Mason's avatar
Chris Mason committed
6693 6694
	err = btrfs_update_inode(trans, root, inode);
	if (err)
6695
		goto out_fail_inode;
6696

6697 6698
	err = btrfs_add_link(trans, dir, inode, dentry->d_name.name,
			     dentry->d_name.len, 0, index);
Chris Mason's avatar
Chris Mason committed
6699
	if (err)
6700
		goto out_fail_inode;
6701

Chris Mason's avatar
Chris Mason committed
6702
	d_instantiate(dentry, inode);
6703 6704 6705 6706 6707
	/*
	 * mkdir is special.  We're unlocking after we call d_instantiate
	 * to avoid a race with nfsd calling d_instantiate.
	 */
	unlock_new_inode(inode);
Chris Mason's avatar
Chris Mason committed
6708 6709 6710
	drop_on_err = 0;

out_fail:
6711
	btrfs_end_transaction(trans, root);
6712 6713
	if (drop_on_err) {
		inode_dec_link_count(inode);
Chris Mason's avatar
Chris Mason committed
6714
		iput(inode);
6715
	}
6716
	btrfs_balance_delayed_items(root);
6717
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
6718
	return err;
6719 6720 6721 6722

out_fail_inode:
	unlock_new_inode(inode);
	goto out_fail;
Chris Mason's avatar
Chris Mason committed
6723 6724
}

6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745
/* Find next extent map of a given extent map, caller needs to ensure locks */
static struct extent_map *next_extent_map(struct extent_map *em)
{
	struct rb_node *next;

	next = rb_next(&em->rb_node);
	if (!next)
		return NULL;
	return container_of(next, struct extent_map, rb_node);
}

static struct extent_map *prev_extent_map(struct extent_map *em)
{
	struct rb_node *prev;

	prev = rb_prev(&em->rb_node);
	if (!prev)
		return NULL;
	return container_of(prev, struct extent_map, rb_node);
}

6746
/* helper for btfs_get_extent.  Given an existing extent in the tree,
6747
 * the existing extent is the nearest extent to map_start,
6748
 * and an extent that you want to insert, deal with overlap and insert
6749
 * the best fitted new extent into the tree.
6750
 */
6751 6752
static int merge_extent_mapping(struct extent_map_tree *em_tree,
				struct extent_map *existing,
6753
				struct extent_map *em,
6754
				u64 map_start)
6755
{
6756 6757 6758 6759
	struct extent_map *prev;
	struct extent_map *next;
	u64 start;
	u64 end;
6760 6761
	u64 start_diff;

6762
	BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778

	if (existing->start > map_start) {
		next = existing;
		prev = prev_extent_map(next);
	} else {
		prev = existing;
		next = next_extent_map(prev);
	}

	start = prev ? extent_map_end(prev) : em->start;
	start = max_t(u64, start, em->start);
	end = next ? next->start : extent_map_end(em);
	end = min_t(u64, end, extent_map_end(em));
	start_diff = start - em->start;
	em->start = start;
	em->len = end - start;
6779 6780
	if (em->block_start < EXTENT_MAP_LAST_BYTE &&
	    !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
6781
		em->block_start += start_diff;
6782 6783
		em->block_len -= start_diff;
	}
Josef Bacik's avatar
Josef Bacik committed
6784
	return add_extent_mapping(em_tree, em, 0);
6785 6786
}

6787
static noinline int uncompress_inline(struct btrfs_path *path,
6788
				      struct page *page,
6789 6790 6791 6792 6793 6794 6795 6796 6797
				      size_t pg_offset, u64 extent_offset,
				      struct btrfs_file_extent_item *item)
{
	int ret;
	struct extent_buffer *leaf = path->nodes[0];
	char *tmp;
	size_t max_size;
	unsigned long inline_size;
	unsigned long ptr;
6798
	int compress_type;
6799 6800

	WARN_ON(pg_offset != 0);
6801
	compress_type = btrfs_file_extent_compression(leaf, item);
6802 6803
	max_size = btrfs_file_extent_ram_bytes(leaf, item);
	inline_size = btrfs_file_extent_inline_item_len(leaf,
6804
					btrfs_item_nr(path->slots[0]));
6805
	tmp = kmalloc(inline_size, GFP_NOFS);
6806 6807
	if (!tmp)
		return -ENOMEM;
6808 6809 6810 6811
	ptr = btrfs_file_extent_inline_start(item);

	read_extent_buffer(leaf, tmp, ptr, inline_size);

6812
	max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6813 6814
	ret = btrfs_decompress(compress_type, tmp, page,
			       extent_offset, inline_size, max_size);
6815
	kfree(tmp);
6816
	return ret;
6817 6818
}

6819 6820
/*
 * a bit scary, this does extent mapping from logical file offset to the disk.
6821 6822
 * the ugly parts come from merging extents from the disk with the in-ram
 * representation.  This gets more complex because of the data=ordered code,
6823 6824 6825 6826
 * where the in-ram extents might be locked pending data=ordered completion.
 *
 * This also copies inline extents directly into the page.
 */
6827

6828
struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
6829
				    size_t pg_offset, u64 start, u64 len,
6830 6831 6832 6833 6834 6835
				    int create)
{
	int ret;
	int err = 0;
	u64 extent_start = 0;
	u64 extent_end = 0;
6836
	u64 objectid = btrfs_ino(inode);
6837
	u32 found_type;
6838
	struct btrfs_path *path = NULL;
6839 6840
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_file_extent_item *item;
6841 6842
	struct extent_buffer *leaf;
	struct btrfs_key found_key;
6843 6844
	struct extent_map *em = NULL;
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
6845
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
6846
	struct btrfs_trans_handle *trans = NULL;
6847
	const bool new_inline = !page || create;
6848 6849

again:
6850
	read_lock(&em_tree->lock);
6851
	em = lookup_extent_mapping(em_tree, start, len);
6852 6853
	if (em)
		em->bdev = root->fs_info->fs_devices->latest_bdev;
6854
	read_unlock(&em_tree->lock);
6855

6856
	if (em) {
6857 6858 6859
		if (em->start > start || em->start + em->len <= start)
			free_extent_map(em);
		else if (em->block_start == EXTENT_MAP_INLINE && page)
6860 6861 6862
			free_extent_map(em);
		else
			goto out;
6863
	}
6864
	em = alloc_extent_map();
6865
	if (!em) {
6866 6867
		err = -ENOMEM;
		goto out;
6868
	}
6869
	em->bdev = root->fs_info->fs_devices->latest_bdev;
6870
	em->start = EXTENT_MAP_HOLE;
6871
	em->orig_start = EXTENT_MAP_HOLE;
6872
	em->len = (u64)-1;
6873
	em->block_len = (u64)-1;
6874 6875 6876

	if (!path) {
		path = btrfs_alloc_path();
6877 6878 6879 6880 6881 6882 6883 6884
		if (!path) {
			err = -ENOMEM;
			goto out;
		}
		/*
		 * Chances are we'll be called again, so go ahead and do
		 * readahead
		 */
6885
		path->reada = READA_FORWARD;
6886 6887
	}

6888 6889
	ret = btrfs_lookup_file_extent(trans, root, path,
				       objectid, start, trans != NULL);
6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900
	if (ret < 0) {
		err = ret;
		goto out;
	}

	if (ret != 0) {
		if (path->slots[0] == 0)
			goto not_found;
		path->slots[0]--;
	}

6901 6902
	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0],
6903 6904
			      struct btrfs_file_extent_item);
	/* are we inside the extent that was found? */
6905
	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6906
	found_type = found_key.type;
6907
	if (found_key.objectid != objectid ||
6908
	    found_type != BTRFS_EXTENT_DATA_KEY) {
6909 6910 6911 6912 6913 6914 6915 6916
		/*
		 * If we backup past the first extent we want to move forward
		 * and see if there is an extent in front of us, otherwise we'll
		 * say there is a hole for our whole search range which can
		 * cause problems.
		 */
		extent_end = start;
		goto next;
6917 6918
	}

6919 6920
	found_type = btrfs_file_extent_type(leaf, item);
	extent_start = found_key.offset;
Yan Zheng's avatar
Yan Zheng committed
6921 6922
	if (found_type == BTRFS_FILE_EXTENT_REG ||
	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6923
		extent_end = extent_start +
6924
		       btrfs_file_extent_num_bytes(leaf, item);
Yan Zheng's avatar
Yan Zheng committed
6925 6926
	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
		size_t size;
6927
		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6928
		extent_end = ALIGN(extent_start + size, root->sectorsize);
Yan Zheng's avatar
Yan Zheng committed
6929
	}
6930
next:
Yan Zheng's avatar
Yan Zheng committed
6931 6932 6933 6934 6935 6936 6937
	if (start >= extent_end) {
		path->slots[0]++;
		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0) {
				err = ret;
				goto out;
6938
			}
Yan Zheng's avatar
Yan Zheng committed
6939 6940 6941
			if (ret > 0)
				goto not_found;
			leaf = path->nodes[0];
6942
		}
Yan Zheng's avatar
Yan Zheng committed
6943 6944 6945 6946 6947 6948
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
		if (found_key.objectid != objectid ||
		    found_key.type != BTRFS_EXTENT_DATA_KEY)
			goto not_found;
		if (start + len <= found_key.offset)
			goto not_found;
6949 6950
		if (start > found_key.offset)
			goto next;
Yan Zheng's avatar
Yan Zheng committed
6951
		em->start = start;
6952
		em->orig_start = start;
Yan Zheng's avatar
Yan Zheng committed
6953 6954 6955 6956
		em->len = found_key.offset - start;
		goto not_found_em;
	}

6957 6958
	btrfs_extent_item_to_extent_map(inode, path, item, new_inline, em);

Yan Zheng's avatar
Yan Zheng committed
6959 6960
	if (found_type == BTRFS_FILE_EXTENT_REG ||
	    found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6961 6962
		goto insert;
	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6963
		unsigned long ptr;
6964
		char *map;
6965 6966 6967
		size_t size;
		size_t extent_offset;
		size_t copy_size;
6968

6969
		if (new_inline)
6970
			goto out;
6971

6972
		size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
Yan Zheng's avatar
Yan Zheng committed
6973
		extent_offset = page_offset(page) + pg_offset - extent_start;
6974 6975
		copy_size = min_t(u64, PAGE_SIZE - pg_offset,
				  size - extent_offset);
6976
		em->start = extent_start + extent_offset;
6977
		em->len = ALIGN(copy_size, root->sectorsize);
6978
		em->orig_block_len = em->len;
6979
		em->orig_start = em->start;
6980
		ptr = btrfs_file_extent_inline_start(item) + extent_offset;
6981
		if (create == 0 && !PageUptodate(page)) {
6982 6983
			if (btrfs_file_extent_compression(leaf, item) !=
			    BTRFS_COMPRESS_NONE) {
6984
				ret = uncompress_inline(path, page, pg_offset,
6985
							extent_offset, item);
6986 6987 6988 6989
				if (ret) {
					err = ret;
					goto out;
				}
6990 6991 6992 6993
			} else {
				map = kmap(page);
				read_extent_buffer(leaf, map + pg_offset, ptr,
						   copy_size);
6994
				if (pg_offset + copy_size < PAGE_SIZE) {
6995
					memset(map + pg_offset + copy_size, 0,
6996
					       PAGE_SIZE - pg_offset -
6997 6998
					       copy_size);
				}
6999 7000
				kunmap(page);
			}
7001 7002
			flush_dcache_page(page);
		} else if (create && PageUptodate(page)) {
7003
			BUG();
7004 7005 7006 7007
			if (!trans) {
				kunmap(page);
				free_extent_map(em);
				em = NULL;
Chris Mason's avatar
Chris Mason committed
7008

7009
				btrfs_release_path(path);
7010
				trans = btrfs_join_transaction(root);
Chris Mason's avatar
Chris Mason committed
7011

7012 7013
				if (IS_ERR(trans))
					return ERR_CAST(trans);
7014 7015
				goto again;
			}
7016
			map = kmap(page);
7017
			write_extent_buffer(leaf, map + pg_offset, ptr,
7018
					    copy_size);
7019
			kunmap(page);
7020
			btrfs_mark_buffer_dirty(leaf);
7021
		}
7022
		set_extent_uptodate(io_tree, em->start,
7023
				    extent_map_end(em) - 1, NULL, GFP_NOFS);
7024 7025 7026 7027
		goto insert;
	}
not_found:
	em->start = start;
7028
	em->orig_start = start;
7029
	em->len = len;
7030
not_found_em:
7031
	em->block_start = EXTENT_MAP_HOLE;
Yan Zheng's avatar
Yan Zheng committed
7032
	set_bit(EXTENT_FLAG_VACANCY, &em->flags);
7033
insert:
7034
	btrfs_release_path(path);
7035
	if (em->start > start || extent_map_end(em) <= start) {
7036 7037 7038
		btrfs_err(root->fs_info,
			  "bad extent! em: [%llu %llu] passed [%llu %llu]",
			  em->start, em->len, start, len);
7039 7040 7041
		err = -EIO;
		goto out;
	}
7042 7043

	err = 0;
7044
	write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
7045
	ret = add_extent_mapping(em_tree, em, 0);
7046 7047 7048 7049
	/* it is possible that someone inserted the extent into the tree
	 * while we had the lock dropped.  It is also possible that
	 * an overlapping map exists in the tree
	 */
7050
	if (ret == -EEXIST) {
7051
		struct extent_map *existing;
7052 7053 7054

		ret = 0;

7055 7056 7057 7058 7059
		existing = search_extent_mapping(em_tree, start, len);
		/*
		 * existing will always be non-NULL, since there must be
		 * extent causing the -EEXIST.
		 */
7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071
		if (existing->start == em->start &&
		    extent_map_end(existing) == extent_map_end(em) &&
		    em->block_start == existing->block_start) {
			/*
			 * these two extents are the same, it happens
			 * with inlines especially
			 */
			free_extent_map(em);
			em = existing;
			err = 0;

		} else if (start >= extent_map_end(existing) ||
7072
		    start <= existing->start) {
7073 7074 7075 7076 7077 7078
			/*
			 * The existing extent map is the one nearest to
			 * the [start, start + len) range which overlaps
			 */
			err = merge_extent_mapping(em_tree, existing,
						   em, start);
7079
			free_extent_map(existing);
7080
			if (err) {
7081 7082 7083 7084 7085 7086
				free_extent_map(em);
				em = NULL;
			}
		} else {
			free_extent_map(em);
			em = existing;
7087
			err = 0;
7088 7089
		}
	}
7090
	write_unlock(&em_tree->lock);
7091
out:
7092

7093
	trace_btrfs_get_extent(root, em);
7094

7095
	btrfs_free_path(path);
7096 7097
	if (trans) {
		ret = btrfs_end_transaction(trans, root);
7098
		if (!err)
7099 7100 7101 7102 7103 7104
			err = ret;
	}
	if (err) {
		free_extent_map(em);
		return ERR_PTR(err);
	}
7105
	BUG_ON(!em); /* Error is always set */
7106 7107 7108
	return em;
}

7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125
struct extent_map *btrfs_get_extent_fiemap(struct inode *inode, struct page *page,
					   size_t pg_offset, u64 start, u64 len,
					   int create)
{
	struct extent_map *em;
	struct extent_map *hole_em = NULL;
	u64 range_start = start;
	u64 end;
	u64 found;
	u64 found_end;
	int err = 0;

	em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
	if (IS_ERR(em))
		return em;
	if (em) {
		/*
7126 7127 7128 7129
		 * if our em maps to
		 * -  a hole or
		 * -  a pre-alloc extent,
		 * there might actually be delalloc bytes behind it.
7130
		 */
7131 7132
		if (em->block_start != EXTENT_MAP_HOLE &&
		    !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166
			return em;
		else
			hole_em = em;
	}

	/* check to see if we've wrapped (len == -1 or similar) */
	end = start + len;
	if (end < start)
		end = (u64)-1;
	else
		end -= 1;

	em = NULL;

	/* ok, we didn't find anything, lets look for delalloc */
	found = count_range_bits(&BTRFS_I(inode)->io_tree, &range_start,
				 end, len, EXTENT_DELALLOC, 1);
	found_end = range_start + found;
	if (found_end < range_start)
		found_end = (u64)-1;

	/*
	 * we didn't find anything useful, return
	 * the original results from get_extent()
	 */
	if (range_start > end || found_end <= start) {
		em = hole_em;
		hole_em = NULL;
		goto out;
	}

	/* adjust the range_start to make sure it doesn't
	 * go backwards from the start they passed in
	 */
7167
	range_start = max(start, range_start);
7168 7169 7170 7171 7172 7173
	found = found_end - range_start;

	if (found > 0) {
		u64 hole_start = start;
		u64 hole_len = len;

7174
		em = alloc_extent_map();
7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213
		if (!em) {
			err = -ENOMEM;
			goto out;
		}
		/*
		 * when btrfs_get_extent can't find anything it
		 * returns one huge hole
		 *
		 * make sure what it found really fits our range, and
		 * adjust to make sure it is based on the start from
		 * the caller
		 */
		if (hole_em) {
			u64 calc_end = extent_map_end(hole_em);

			if (calc_end <= start || (hole_em->start > end)) {
				free_extent_map(hole_em);
				hole_em = NULL;
			} else {
				hole_start = max(hole_em->start, start);
				hole_len = calc_end - hole_start;
			}
		}
		em->bdev = NULL;
		if (hole_em && range_start > hole_start) {
			/* our hole starts before our delalloc, so we
			 * have to return just the parts of the hole
			 * that go until  the delalloc starts
			 */
			em->len = min(hole_len,
				      range_start - hole_start);
			em->start = hole_start;
			em->orig_start = hole_start;
			/*
			 * don't adjust block start at all,
			 * it is fixed at EXTENT_MAP_HOLE
			 */
			em->block_start = hole_em->block_start;
			em->block_len = hole_len;
7214 7215
			if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
				set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235
		} else {
			em->start = range_start;
			em->len = found;
			em->orig_start = range_start;
			em->block_start = EXTENT_MAP_DELALLOC;
			em->block_len = found;
		}
	} else if (hole_em) {
		return hole_em;
	}
out:

	free_extent_map(hole_em);
	if (err) {
		free_extent_map(em);
		return ERR_PTR(err);
	}
	return em;
}

7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270
static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
						  const u64 start,
						  const u64 len,
						  const u64 orig_start,
						  const u64 block_start,
						  const u64 block_len,
						  const u64 orig_block_len,
						  const u64 ram_bytes,
						  const int type)
{
	struct extent_map *em = NULL;
	int ret;

	if (type != BTRFS_ORDERED_NOCOW) {
		em = create_pinned_em(inode, start, len, orig_start,
				      block_start, block_len, orig_block_len,
				      ram_bytes, type);
		if (IS_ERR(em))
			goto out;
	}
	ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
					   len, block_len, type);
	if (ret) {
		if (em) {
			free_extent_map(em);
			btrfs_drop_extent_cache(inode, start,
						start + len - 1, 0);
		}
		em = ERR_PTR(ret);
	}
 out:

	return em;
}

7271 7272 7273 7274
static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
						  u64 start, u64 len)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
7275
	struct extent_map *em;
7276 7277 7278 7279 7280
	struct btrfs_key ins;
	u64 alloc_hint;
	int ret;

	alloc_hint = get_extent_allocation_hint(inode, start, len);
7281
	ret = btrfs_reserve_extent(root, len, len, root->sectorsize, 0,
7282
				   alloc_hint, &ins, 1, 1);
7283 7284
	if (ret)
		return ERR_PTR(ret);
7285

7286 7287 7288
	em = btrfs_create_dio_extent(inode, start, ins.offset, start,
				     ins.objectid, ins.offset, ins.offset,
				     ins.offset, 0);
7289
	btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);
7290
	if (IS_ERR(em))
7291
		btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 1);
7292

7293 7294 7295
	return em;
}

7296 7297 7298 7299
/*
 * returns 1 when the nocow is safe, < 1 on error, 0 if the
 * block must be cow'd
 */
7300
noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7301 7302
			      u64 *orig_start, u64 *orig_block_len,
			      u64 *ram_bytes)
7303
{
7304
	struct btrfs_trans_handle *trans;
7305 7306 7307 7308
	struct btrfs_path *path;
	int ret;
	struct extent_buffer *leaf;
	struct btrfs_root *root = BTRFS_I(inode)->root;
7309
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7310 7311 7312 7313 7314 7315 7316 7317
	struct btrfs_file_extent_item *fi;
	struct btrfs_key key;
	u64 disk_bytenr;
	u64 backref_offset;
	u64 extent_end;
	u64 num_bytes;
	int slot;
	int found_type;
7318
	bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7319

7320 7321 7322 7323
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

7324
	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340
				       offset, 0);
	if (ret < 0)
		goto out;

	slot = path->slots[0];
	if (ret == 1) {
		if (slot == 0) {
			/* can't find the item, must cow */
			ret = 0;
			goto out;
		}
		slot--;
	}
	ret = 0;
	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, slot);
7341
	if (key.objectid != btrfs_ino(inode) ||
7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358
	    key.type != BTRFS_EXTENT_DATA_KEY) {
		/* not our file or wrong item type, must cow */
		goto out;
	}

	if (key.offset > offset) {
		/* Wrong offset, must cow */
		goto out;
	}

	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
	found_type = btrfs_file_extent_type(leaf, fi);
	if (found_type != BTRFS_FILE_EXTENT_REG &&
	    found_type != BTRFS_FILE_EXTENT_PREALLOC) {
		/* not a regular extent, must cow */
		goto out;
	}
7359 7360 7361 7362

	if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
		goto out;

7363 7364 7365 7366
	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
	if (extent_end <= offset)
		goto out;

7367
	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7368 7369 7370 7371 7372 7373 7374 7375
	if (disk_bytenr == 0)
		goto out;

	if (btrfs_file_extent_compression(leaf, fi) ||
	    btrfs_file_extent_encryption(leaf, fi) ||
	    btrfs_file_extent_other_encoding(leaf, fi))
		goto out;

7376 7377
	backref_offset = btrfs_file_extent_offset(leaf, fi);

7378 7379 7380 7381 7382
	if (orig_start) {
		*orig_start = key.offset - backref_offset;
		*orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
		*ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
	}
7383

7384 7385
	if (btrfs_extent_readonly(root, disk_bytenr))
		goto out;
7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399

	num_bytes = min(offset + *len, extent_end) - offset;
	if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
		u64 range_end;

		range_end = round_up(offset + num_bytes, root->sectorsize) - 1;
		ret = test_range_bit(io_tree, offset, range_end,
				     EXTENT_DELALLOC, 0, NULL);
		if (ret) {
			ret = -EAGAIN;
			goto out;
		}
	}

7400
	btrfs_release_path(path);
7401 7402 7403 7404 7405

	/*
	 * look for other files referencing this extent, if we
	 * find any we must cow
	 */
7406 7407 7408
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans)) {
		ret = 0;
7409
		goto out;
7410 7411 7412 7413 7414 7415 7416 7417 7418
	}

	ret = btrfs_cross_ref_exist(trans, root, btrfs_ino(inode),
				    key.offset - backref_offset, disk_bytenr);
	btrfs_end_transaction(trans, root);
	if (ret) {
		ret = 0;
		goto out;
	}
7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433

	/*
	 * adjust disk_bytenr and num_bytes to cover just the bytes
	 * in this extent we are about to write.  If there
	 * are any csums in that range we have to cow in order
	 * to keep the csums correct
	 */
	disk_bytenr += backref_offset;
	disk_bytenr += offset - key.offset;
	if (csum_exist_in_range(root, disk_bytenr, num_bytes))
				goto out;
	/*
	 * all of the above have passed, it is safe to overwrite this extent
	 * without cow
	 */
7434
	*len = num_bytes;
7435 7436 7437 7438 7439 7440
	ret = 1;
out:
	btrfs_free_path(path);
	return ret;
}

7441 7442 7443 7444 7445 7446
bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
{
	struct radix_tree_root *root = &inode->i_mapping->page_tree;
	int found = false;
	void **pagep = NULL;
	struct page *page = NULL;
7447 7448
	unsigned long start_idx;
	unsigned long end_idx;
7449

7450
	start_idx = start >> PAGE_SHIFT;
7451 7452 7453 7454

	/*
	 * end is the last byte in the last page.  end == start is legal
	 */
7455
	end_idx = end >> PAGE_SHIFT;
7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471

	rcu_read_lock();

	/* Most of the code in this while loop is lifted from
	 * find_get_page.  It's been modified to begin searching from a
	 * page and return just the first page found in that range.  If the
	 * found idx is less than or equal to the end idx then we know that
	 * a page exists.  If no pages are found or if those pages are
	 * outside of the range then we're fine (yay!) */
	while (page == NULL &&
	       radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
		page = radix_tree_deref_slot(pagep);
		if (unlikely(!page))
			break;

		if (radix_tree_exception(page)) {
7472 7473
			if (radix_tree_deref_retry(page)) {
				page = NULL;
7474
				continue;
7475
			}
7476 7477 7478 7479 7480
			/*
			 * Otherwise, shmem/tmpfs must be storing a swap entry
			 * here as an exceptional entry: so return it without
			 * attempting to raise page count.
			 */
7481
			page = NULL;
7482 7483 7484
			break; /* TODO: Is this relevant for this use case? */
		}

7485 7486
		if (!page_cache_get_speculative(page)) {
			page = NULL;
7487
			continue;
7488
		}
7489 7490 7491 7492 7493 7494 7495

		/*
		 * Has the page moved?
		 * This is part of the lockless pagecache protocol. See
		 * include/linux/pagemap.h for details.
		 */
		if (unlikely(page != *pagep)) {
7496
			put_page(page);
7497 7498 7499 7500 7501 7502 7503
			page = NULL;
		}
	}

	if (page) {
		if (page->index <= end_idx)
			found = true;
7504
		put_page(page);
7505 7506 7507 7508 7509 7510
	}

	rcu_read_unlock();
	return found;
}

7511 7512 7513 7514 7515 7516 7517 7518
static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
			      struct extent_state **cached_state, int writing)
{
	struct btrfs_ordered_extent *ordered;
	int ret = 0;

	while (1) {
		lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7519
				 cached_state);
7520 7521
		/*
		 * We're concerned with the entire range that we're going to be
7522
		 * doing DIO to, so we need to make sure there's no ordered
7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534
		 * extents in this range.
		 */
		ordered = btrfs_lookup_ordered_range(inode, lockstart,
						     lockend - lockstart + 1);

		/*
		 * We need to make sure there are no buffered pages in this
		 * range either, we could have raced between the invalidate in
		 * generic_file_direct_write and locking the extent.  The
		 * invalidate needs to happen so that reads after a write do not
		 * get stale data.
		 */
7535 7536 7537
		if (!ordered &&
		    (!writing ||
		     !btrfs_page_exists_in_range(inode, lockstart, lockend)))
7538 7539 7540 7541 7542 7543
			break;

		unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
				     cached_state, GFP_NOFS);

		if (ordered) {
7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563
			/*
			 * If we are doing a DIO read and the ordered extent we
			 * found is for a buffered write, we can not wait for it
			 * to complete and retry, because if we do so we can
			 * deadlock with concurrent buffered writes on page
			 * locks. This happens only if our DIO read covers more
			 * than one extent map, if at this point has already
			 * created an ordered extent for a previous extent map
			 * and locked its range in the inode's io tree, and a
			 * concurrent write against that previous extent map's
			 * range and this range started (we unlock the ranges
			 * in the io tree only when the bios complete and
			 * buffered writes always lock pages before attempting
			 * to lock range in the io tree).
			 */
			if (writing ||
			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
				btrfs_start_ordered_extent(inode, ordered, 1);
			else
				ret = -ENOTBLK;
7564 7565 7566
			btrfs_put_ordered_extent(ordered);
		} else {
			/*
7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577
			 * We could trigger writeback for this range (and wait
			 * for it to complete) and then invalidate the pages for
			 * this range (through invalidate_inode_pages2_range()),
			 * but that can lead us to a deadlock with a concurrent
			 * call to readpages() (a buffered read or a defrag call
			 * triggered a readahead) on a page lock due to an
			 * ordered dio extent we created before but did not have
			 * yet a corresponding bio submitted (whence it can not
			 * complete), which makes readpages() wait for that
			 * ordered extent to complete while holding a lock on
			 * that page.
7578
			 */
7579
			ret = -ENOTBLK;
7580 7581
		}

7582 7583 7584
		if (ret)
			break;

7585 7586 7587 7588 7589 7590
		cond_resched();
	}

	return ret;
}

7591 7592 7593
static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
					   u64 len, u64 orig_start,
					   u64 block_start, u64 block_len,
Josef Bacik's avatar
Josef Bacik committed
7594 7595
					   u64 orig_block_len, u64 ram_bytes,
					   int type)
7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608
{
	struct extent_map_tree *em_tree;
	struct extent_map *em;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	int ret;

	em_tree = &BTRFS_I(inode)->extent_tree;
	em = alloc_extent_map();
	if (!em)
		return ERR_PTR(-ENOMEM);

	em->start = start;
	em->orig_start = orig_start;
7609 7610
	em->mod_start = start;
	em->mod_len = len;
7611 7612 7613 7614
	em->len = len;
	em->block_len = block_len;
	em->block_start = block_start;
	em->bdev = root->fs_info->fs_devices->latest_bdev;
7615
	em->orig_block_len = orig_block_len;
Josef Bacik's avatar
Josef Bacik committed
7616
	em->ram_bytes = ram_bytes;
7617
	em->generation = -1;
7618 7619
	set_bit(EXTENT_FLAG_PINNED, &em->flags);
	if (type == BTRFS_ORDERED_PREALLOC)
7620
		set_bit(EXTENT_FLAG_FILLING, &em->flags);
7621 7622 7623 7624 7625

	do {
		btrfs_drop_extent_cache(inode, em->start,
				em->start + em->len - 1, 0);
		write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
7626
		ret = add_extent_mapping(em_tree, em, 1);
7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637
		write_unlock(&em_tree->lock);
	} while (ret == -EEXIST);

	if (ret) {
		free_extent_map(em);
		return ERR_PTR(ret);
	}

	return em;
}

7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650
static void adjust_dio_outstanding_extents(struct inode *inode,
					   struct btrfs_dio_data *dio_data,
					   const u64 len)
{
	unsigned num_extents;

	num_extents = (unsigned) div64_u64(len + BTRFS_MAX_EXTENT_SIZE - 1,
					   BTRFS_MAX_EXTENT_SIZE);
	/*
	 * If we have an outstanding_extents count still set then we're
	 * within our reservation, otherwise we need to adjust our inode
	 * counter appropriately.
	 */
7651
	if (dio_data->outstanding_extents >= num_extents) {
7652 7653
		dio_data->outstanding_extents -= num_extents;
	} else {
7654 7655 7656 7657 7658 7659 7660
		/*
		 * If dio write length has been split due to no large enough
		 * contiguous space, we need to compensate our inode counter
		 * appropriately.
		 */
		u64 num_needed = num_extents - dio_data->outstanding_extents;

7661
		spin_lock(&BTRFS_I(inode)->lock);
7662
		BTRFS_I(inode)->outstanding_extents += num_needed;
7663 7664 7665 7666
		spin_unlock(&BTRFS_I(inode)->lock);
	}
}

7667 7668 7669 7670 7671
static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
				   struct buffer_head *bh_result, int create)
{
	struct extent_map *em;
	struct btrfs_root *root = BTRFS_I(inode)->root;
7672
	struct extent_state *cached_state = NULL;
7673
	struct btrfs_dio_data *dio_data = NULL;
7674
	u64 start = iblock << inode->i_blkbits;
7675
	u64 lockstart, lockend;
7676
	u64 len = bh_result->b_size;
7677
	int unlock_bits = EXTENT_LOCKED;
7678
	int ret = 0;
7679

7680
	if (create)
7681
		unlock_bits |= EXTENT_DIRTY;
7682
	else
7683
		len = min_t(u64, len, root->sectorsize);
7684

7685 7686 7687
	lockstart = start;
	lockend = start + len - 1;

7688 7689 7690
	if (current->journal_info) {
		/*
		 * Need to pull our outstanding extents and set journal_info to NULL so
7691
		 * that anything that needs to check if there's a transaction doesn't get
7692 7693
		 * confused.
		 */
7694
		dio_data = current->journal_info;
7695 7696 7697
		current->journal_info = NULL;
	}

7698 7699 7700 7701
	/*
	 * If this errors out it's because we couldn't invalidate pagecache for
	 * this range and we need to fallback to buffered.
	 */
7702 7703 7704 7705 7706
	if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
			       create)) {
		ret = -ENOTBLK;
		goto err;
	}
7707

7708
	em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
7709 7710 7711 7712
	if (IS_ERR(em)) {
		ret = PTR_ERR(em);
		goto unlock_err;
	}
7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723

	/*
	 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
	 * io.  INLINE is special, and we could probably kludge it in here, but
	 * it's still buffered so for safety lets just fall back to the generic
	 * buffered path.
	 *
	 * For COMPRESSED we _have_ to read the entire extent in so we can
	 * decompress it, so there will be buffering required no matter what we
	 * do, so go ahead and fallback to buffered.
	 *
7724
	 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7725 7726 7727 7728 7729 7730
	 * to buffered IO.  Don't blame me, this is the price we pay for using
	 * the generic code.
	 */
	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
	    em->block_start == EXTENT_MAP_INLINE) {
		free_extent_map(em);
7731 7732
		ret = -ENOTBLK;
		goto unlock_err;
7733 7734 7735 7736 7737 7738
	}

	/* Just a good old fashioned hole, return */
	if (!create && (em->block_start == EXTENT_MAP_HOLE ||
			test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
		free_extent_map(em);
7739
		goto unlock_err;
7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750
	}

	/*
	 * We don't allocate a new extent in the following cases
	 *
	 * 1) The inode is marked as NODATACOW.  In this case we'll just use the
	 * existing extent.
	 * 2) The extent is marked as PREALLOC.  We're good to go here and can
	 * just use the extent.
	 *
	 */
7751
	if (!create) {
7752 7753 7754
		len = min(len, em->len - (start - em->start));
		lockstart = start + len;
		goto unlock;
7755
	}
7756 7757 7758 7759 7760

	if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
	    ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
	     em->block_start != EXTENT_MAP_HOLE)) {
		int type;
7761
		u64 block_start, orig_start, orig_block_len, ram_bytes;
7762 7763 7764 7765 7766

		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
			type = BTRFS_ORDERED_PREALLOC;
		else
			type = BTRFS_ORDERED_NOCOW;
7767
		len = min(len, em->len - (start - em->start));
7768
		block_start = em->block_start + (start - em->start);
7769

7770
		if (can_nocow_extent(inode, start, &len, &orig_start,
7771 7772
				     &orig_block_len, &ram_bytes) == 1 &&
		    btrfs_inc_nocow_writers(root->fs_info, block_start)) {
7773
			struct extent_map *em2;
7774

7775 7776 7777 7778
			em2 = btrfs_create_dio_extent(inode, start, len,
						      orig_start, block_start,
						      len, orig_block_len,
						      ram_bytes, type);
7779
			btrfs_dec_nocow_writers(root->fs_info, block_start);
7780 7781
			if (type == BTRFS_ORDERED_PREALLOC) {
				free_extent_map(em);
7782
				em = em2;
7783
			}
7784 7785
			if (em2 && IS_ERR(em2)) {
				ret = PTR_ERR(em2);
7786
				goto unlock_err;
7787
			}
7788 7789 7790 7791 7792 7793 7794
			/*
			 * For inode marked NODATACOW or extent marked PREALLOC,
			 * use the existing or preallocated extent, so does not
			 * need to adjust btrfs_space_info's bytes_may_use.
			 */
			btrfs_free_reserved_data_space_noquota(inode,
					start, len);
7795
			goto unlock;
7796 7797
		}
	}
7798

7799 7800 7801 7802 7803
	/*
	 * this will cow the extent, reset the len in case we changed
	 * it above
	 */
	len = bh_result->b_size;
7804 7805
	free_extent_map(em);
	em = btrfs_new_extent_direct(inode, start, len);
7806 7807 7808 7809
	if (IS_ERR(em)) {
		ret = PTR_ERR(em);
		goto unlock_err;
	}
7810 7811
	len = min(len, em->len - (start - em->start));
unlock:
7812 7813
	bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
		inode->i_blkbits;
7814
	bh_result->b_size = len;
7815 7816
	bh_result->b_bdev = em->bdev;
	set_buffer_mapped(bh_result);
7817 7818 7819 7820 7821 7822 7823 7824 7825 7826
	if (create) {
		if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
			set_buffer_new(bh_result);

		/*
		 * Need to update the i_size under the extent lock so buffered
		 * readers will get the updated i_size when we unlock.
		 */
		if (start + len > i_size_read(inode))
			i_size_write(inode, start + len);
7827

7828
		adjust_dio_outstanding_extents(inode, dio_data, len);
7829 7830
		WARN_ON(dio_data->reserve < len);
		dio_data->reserve -= len;
7831
		dio_data->unsubmitted_oe_range_end = start + len;
7832
		current->journal_info = dio_data;
7833
	}
7834

7835 7836 7837 7838 7839
	/*
	 * In the case of write we need to clear and unlock the entire range,
	 * in the case of read we need to unlock only the end area that we
	 * aren't using if there is any left over space.
	 */
Liu Bo's avatar
Liu Bo committed
7840
	if (lockstart < lockend) {
7841 7842 7843
		clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
				 lockend, unlock_bits, 1, 0,
				 &cached_state, GFP_NOFS);
Liu Bo's avatar
Liu Bo committed
7844
	} else {
7845
		free_extent_state(cached_state);
Liu Bo's avatar
Liu Bo committed
7846
	}
7847

7848 7849 7850
	free_extent_map(em);

	return 0;
7851 7852 7853 7854

unlock_err:
	clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
			 unlock_bits, 1, 0, &cached_state, GFP_NOFS);
7855
err:
7856 7857
	if (dio_data)
		current->journal_info = dio_data;
7858 7859 7860 7861 7862 7863 7864 7865
	/*
	 * Compensate the delalloc release we do in btrfs_direct_IO() when we
	 * write less data then expected, so that we don't underflow our inode's
	 * outstanding extents counter.
	 */
	if (create && dio_data)
		adjust_dio_outstanding_extents(inode, dio_data, len);

7866
	return ret;
7867 7868
}

7869
static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,
7870
					int mirror_num)
7871 7872 7873 7874
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	int ret;

7875
	BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7876 7877 7878 7879 7880 7881 7882 7883

	bio_get(bio);

	ret = btrfs_bio_wq_end_io(root->fs_info, bio,
				  BTRFS_WQ_ENDIO_DIO_REPAIR);
	if (ret)
		goto err;

7884
	ret = btrfs_map_bio(root, bio, mirror_num, 0);
7885 7886 7887 7888 7889 7890 7891 7892 7893 7894
err:
	bio_put(bio);
	return ret;
}

static int btrfs_check_dio_repairable(struct inode *inode,
				      struct bio *failed_bio,
				      struct io_failure_record *failrec,
				      int failed_mirror)
{
7895
	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7896 7897
	int num_copies;

7898
	num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7899 7900 7901 7902 7903 7904
	if (num_copies == 1) {
		/*
		 * we only have a single copy of the data, so don't bother with
		 * all the retry and error correction code that follows. no
		 * matter what the error is, it is very likely to persist.
		 */
7905 7906 7907
		btrfs_debug(fs_info,
			"Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
			num_copies, failrec->this_mirror, failed_mirror);
7908 7909 7910 7911 7912 7913 7914 7915 7916
		return 0;
	}

	failrec->failed_mirror = failed_mirror;
	failrec->this_mirror++;
	if (failrec->this_mirror == failed_mirror)
		failrec->this_mirror++;

	if (failrec->this_mirror > num_copies) {
7917 7918 7919
		btrfs_debug(fs_info,
			"Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
			num_copies, failrec->this_mirror, failed_mirror);
7920 7921 7922 7923 7924 7925 7926
		return 0;
	}

	return 1;
}

static int dio_read_error(struct inode *inode, struct bio *failed_bio,
7927 7928 7929
			struct page *page, unsigned int pgoff,
			u64 start, u64 end, int failed_mirror,
			bio_end_io_t *repair_endio, void *repair_arg)
7930 7931 7932 7933 7934 7935 7936
{
	struct io_failure_record *failrec;
	struct bio *bio;
	int isector;
	int read_mode;
	int ret;

7937
	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949

	ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
	if (ret)
		return ret;

	ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
					 failed_mirror);
	if (!ret) {
		free_io_failure(inode, failrec);
		return -EIO;
	}

7950 7951 7952
	if ((failed_bio->bi_vcnt > 1)
		|| (failed_bio->bi_io_vec->bv_len
			> BTRFS_I(inode)->root->sectorsize))
7953 7954 7955 7956 7957 7958 7959
		read_mode = READ_SYNC | REQ_FAILFAST_DEV;
	else
		read_mode = READ_SYNC;

	isector = start - btrfs_io_bio(failed_bio)->logical;
	isector >>= inode->i_sb->s_blocksize_bits;
	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
7960
				pgoff, isector, repair_endio, repair_arg);
7961 7962 7963 7964
	if (!bio) {
		free_io_failure(inode, failrec);
		return -EIO;
	}
7965
	bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
7966 7967 7968 7969 7970

	btrfs_debug(BTRFS_I(inode)->root->fs_info,
		    "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n",
		    read_mode, failrec->this_mirror, failrec->in_validation);

7971
	ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986
	if (ret) {
		free_io_failure(inode, failrec);
		bio_put(bio);
	}

	return ret;
}

struct btrfs_retry_complete {
	struct completion done;
	struct inode *inode;
	u64 start;
	int uptodate;
};

7987
static void btrfs_retry_endio_nocsum(struct bio *bio)
7988 7989
{
	struct btrfs_retry_complete *done = bio->bi_private;
7990
	struct inode *inode;
7991 7992 7993
	struct bio_vec *bvec;
	int i;

7994
	if (bio->bi_error)
7995 7996
		goto end;

7997 7998 7999 8000
	ASSERT(bio->bi_vcnt == 1);
	inode = bio->bi_io_vec->bv_page->mapping->host;
	ASSERT(bio->bi_io_vec->bv_len == BTRFS_I(inode)->root->sectorsize);

8001 8002 8003 8004 8005 8006 8007 8008 8009 8010
	done->uptodate = 1;
	bio_for_each_segment_all(bvec, bio, i)
		clean_io_failure(done->inode, done->start, bvec->bv_page, 0);
end:
	complete(&done->done);
	bio_put(bio);
}

static int __btrfs_correct_data_nocsum(struct inode *inode,
				       struct btrfs_io_bio *io_bio)
8011
{
8012
	struct btrfs_fs_info *fs_info;
8013
	struct bio_vec *bvec;
8014
	struct btrfs_retry_complete done;
8015
	u64 start;
8016 8017 8018
	unsigned int pgoff;
	u32 sectorsize;
	int nr_sectors;
8019
	int i;
8020
	int ret;
8021

8022 8023 8024
	fs_info = BTRFS_I(inode)->root->fs_info;
	sectorsize = BTRFS_I(inode)->root->sectorsize;

8025 8026 8027 8028
	start = io_bio->logical;
	done.inode = inode;

	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
8029 8030 8031 8032
		nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);
		pgoff = bvec->bv_offset;

next_block_or_try_again:
8033 8034 8035 8036
		done.uptodate = 0;
		done.start = start;
		init_completion(&done.done);

8037 8038 8039 8040
		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page,
				pgoff, start, start + sectorsize - 1,
				io_bio->mirror_num,
				btrfs_retry_endio_nocsum, &done);
8041 8042 8043 8044 8045 8046 8047
		if (ret)
			return ret;

		wait_for_completion(&done.done);

		if (!done.uptodate) {
			/* We might have another mirror, so try again */
8048
			goto next_block_or_try_again;
8049 8050
		}

8051 8052 8053 8054 8055 8056
		start += sectorsize;

		if (nr_sectors--) {
			pgoff += sectorsize;
			goto next_block_or_try_again;
		}
8057 8058 8059 8060 8061
	}

	return 0;
}

8062
static void btrfs_retry_endio(struct bio *bio)
8063 8064 8065
{
	struct btrfs_retry_complete *done = bio->bi_private;
	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8066
	struct inode *inode;
8067
	struct bio_vec *bvec;
8068
	u64 start;
8069 8070 8071 8072
	int uptodate;
	int ret;
	int i;

8073
	if (bio->bi_error)
8074 8075 8076
		goto end;

	uptodate = 1;
8077 8078 8079 8080 8081 8082 8083

	start = done->start;

	ASSERT(bio->bi_vcnt == 1);
	inode = bio->bi_io_vec->bv_page->mapping->host;
	ASSERT(bio->bi_io_vec->bv_len == BTRFS_I(inode)->root->sectorsize);

8084 8085
	bio_for_each_segment_all(bvec, bio, i) {
		ret = __readpage_endio_check(done->inode, io_bio, i,
8086 8087
					bvec->bv_page, bvec->bv_offset,
					done->start, bvec->bv_len);
8088 8089
		if (!ret)
			clean_io_failure(done->inode, done->start,
8090
					bvec->bv_page, bvec->bv_offset);
8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103
		else
			uptodate = 0;
	}

	done->uptodate = uptodate;
end:
	complete(&done->done);
	bio_put(bio);
}

static int __btrfs_subio_endio_read(struct inode *inode,
				    struct btrfs_io_bio *io_bio, int err)
{
8104
	struct btrfs_fs_info *fs_info;
8105 8106 8107 8108
	struct bio_vec *bvec;
	struct btrfs_retry_complete done;
	u64 start;
	u64 offset = 0;
8109 8110 8111 8112
	u32 sectorsize;
	int nr_sectors;
	unsigned int pgoff;
	int csum_pos;
8113 8114
	int i;
	int ret;
8115

8116 8117 8118
	fs_info = BTRFS_I(inode)->root->fs_info;
	sectorsize = BTRFS_I(inode)->root->sectorsize;

8119
	err = 0;
8120
	start = io_bio->logical;
8121 8122
	done.inode = inode;

8123
	bio_for_each_segment_all(bvec, &io_bio->bio, i) {
8124 8125 8126 8127 8128 8129 8130 8131
		nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len);

		pgoff = bvec->bv_offset;
next_block:
		csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
		ret = __readpage_endio_check(inode, io_bio, csum_pos,
					bvec->bv_page, pgoff, start,
					sectorsize);
8132 8133 8134 8135 8136 8137 8138
		if (likely(!ret))
			goto next;
try_again:
		done.uptodate = 0;
		done.start = start;
		init_completion(&done.done);

8139 8140 8141 8142
		ret = dio_read_error(inode, &io_bio->bio, bvec->bv_page,
				pgoff, start, start + sectorsize - 1,
				io_bio->mirror_num,
				btrfs_retry_endio, &done);
8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154
		if (ret) {
			err = ret;
			goto next;
		}

		wait_for_completion(&done.done);

		if (!done.uptodate) {
			/* We might have another mirror, so try again */
			goto try_again;
		}
next:
8155 8156 8157 8158 8159 8160 8161 8162 8163
		offset += sectorsize;
		start += sectorsize;

		ASSERT(nr_sectors);

		if (--nr_sectors) {
			pgoff += sectorsize;
			goto next_block;
		}
8164
	}
8165 8166 8167 8168

	return err;
}

8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183
static int btrfs_subio_endio_read(struct inode *inode,
				  struct btrfs_io_bio *io_bio, int err)
{
	bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;

	if (skip_csum) {
		if (unlikely(err))
			return __btrfs_correct_data_nocsum(inode, io_bio);
		else
			return 0;
	} else {
		return __btrfs_subio_endio_read(inode, io_bio, err);
	}
}

8184
static void btrfs_endio_direct_read(struct bio *bio)
8185 8186 8187 8188 8189
{
	struct btrfs_dio_private *dip = bio->bi_private;
	struct inode *inode = dip->inode;
	struct bio *dio_bio;
	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8190
	int err = bio->bi_error;
8191

8192 8193
	if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
		err = btrfs_subio_endio_read(inode, io_bio, err);
8194

8195
	unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8196
		      dip->logical_offset + dip->bytes - 1);
8197
	dio_bio = dip->dio_bio;
8198 8199

	kfree(dip);
8200

8201
	dio_bio->bi_error = bio->bi_error;
8202
	dio_end_io(dio_bio, bio->bi_error);
8203 8204 8205

	if (io_bio->end_io)
		io_bio->end_io(io_bio, err);
8206
	bio_put(bio);
8207 8208
}

8209 8210 8211 8212
static void btrfs_endio_direct_write_update_ordered(struct inode *inode,
						    const u64 offset,
						    const u64 bytes,
						    const int uptodate)
8213 8214 8215
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_ordered_extent *ordered = NULL;
8216 8217
	u64 ordered_offset = offset;
	u64 ordered_bytes = bytes;
8218 8219
	int ret;

8220 8221 8222
again:
	ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
						   &ordered_offset,
8223
						   ordered_bytes,
8224
						   uptodate);
8225
	if (!ret)
8226
		goto out_test;
8227

8228 8229
	btrfs_init_work(&ordered->work, btrfs_endio_write_helper,
			finish_ordered_fn, NULL, NULL);
8230 8231
	btrfs_queue_work(root->fs_info->endio_write_workers,
			 &ordered->work);
8232 8233 8234 8235 8236
out_test:
	/*
	 * our bio might span multiple ordered extents.  If we haven't
	 * completed the accounting for the whole dio, go back and try again
	 */
8237 8238
	if (ordered_offset < offset + bytes) {
		ordered_bytes = offset + bytes - ordered_offset;
8239
		ordered = NULL;
8240 8241
		goto again;
	}
8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252
}

static void btrfs_endio_direct_write(struct bio *bio)
{
	struct btrfs_dio_private *dip = bio->bi_private;
	struct bio *dio_bio = dip->dio_bio;

	btrfs_endio_direct_write_update_ordered(dip->inode,
						dip->logical_offset,
						dip->bytes,
						!bio->bi_error);
8253 8254

	kfree(dip);
8255

8256
	dio_bio->bi_error = bio->bi_error;
8257
	dio_end_io(dio_bio, bio->bi_error);
8258
	bio_put(bio);
8259 8260
}

8261
static int __btrfs_submit_bio_start_direct_io(struct inode *inode,
8262 8263 8264 8265 8266 8267
				    struct bio *bio, int mirror_num,
				    unsigned long bio_flags, u64 offset)
{
	int ret;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	ret = btrfs_csum_one_bio(root, inode, bio, offset, 1);
8268
	BUG_ON(ret); /* -ENOMEM */
8269 8270 8271
	return 0;
}

8272
static void btrfs_end_dio_bio(struct bio *bio)
8273 8274
{
	struct btrfs_dio_private *dip = bio->bi_private;
8275
	int err = bio->bi_error;
8276

8277 8278
	if (err)
		btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8279
			   "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8280
			   btrfs_ino(dip->inode), bio_op(bio), bio->bi_opf,
8281 8282 8283 8284 8285
			   (unsigned long long)bio->bi_iter.bi_sector,
			   bio->bi_iter.bi_size, err);

	if (dip->subio_endio)
		err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8286 8287

	if (err) {
8288 8289 8290 8291 8292 8293
		dip->errors = 1;

		/*
		 * before atomic variable goto zero, we must make sure
		 * dip->errors is perceived to be set.
		 */
8294
		smp_mb__before_atomic();
8295 8296 8297 8298 8299 8300
	}

	/* if there are more bios still pending for this dio, just exit */
	if (!atomic_dec_and_test(&dip->pending_bios))
		goto out;

8301
	if (dip->errors) {
8302
		bio_io_error(dip->orig_bio);
8303
	} else {
8304 8305
		dip->dio_bio->bi_error = 0;
		bio_endio(dip->orig_bio);
8306 8307 8308 8309 8310 8311 8312 8313
	}
out:
	bio_put(bio);
}

static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev,
				       u64 first_sector, gfp_t gfp_flags)
{
8314
	struct bio *bio;
8315
	bio = btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags);
8316 8317 8318
	if (bio)
		bio_associate_current(bio);
	return bio;
8319 8320
}

8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352
static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root,
						 struct inode *inode,
						 struct btrfs_dio_private *dip,
						 struct bio *bio,
						 u64 file_offset)
{
	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
	struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
	int ret;

	/*
	 * We load all the csum data we need when we submit
	 * the first bio to reduce the csum tree search and
	 * contention.
	 */
	if (dip->logical_offset == file_offset) {
		ret = btrfs_lookup_bio_sums_dio(root, inode, dip->orig_bio,
						file_offset);
		if (ret)
			return ret;
	}

	if (bio == dip->orig_bio)
		return 0;

	file_offset -= dip->logical_offset;
	file_offset >>= inode->i_sb->s_blocksize_bits;
	io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);

	return 0;
}

8353
static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,
8354
					 u64 file_offset, int skip_sum,
8355
					 int async_submit)
8356
{
8357
	struct btrfs_dio_private *dip = bio->bi_private;
8358
	bool write = bio_op(bio) == REQ_OP_WRITE;
8359 8360 8361
	struct btrfs_root *root = BTRFS_I(inode)->root;
	int ret;

8362 8363 8364
	if (async_submit)
		async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);

8365
	bio_get(bio);
8366 8367

	if (!write) {
8368 8369
		ret = btrfs_bio_wq_end_io(root->fs_info, bio,
				BTRFS_WQ_ENDIO_DATA);
8370 8371 8372
		if (ret)
			goto err;
	}
8373

8374 8375 8376 8377
	if (skip_sum)
		goto map;

	if (write && async_submit) {
8378
		ret = btrfs_wq_submit_bio(root->fs_info,
8379
				   inode, bio, 0, 0, file_offset,
8380 8381 8382
				   __btrfs_submit_bio_start_direct_io,
				   __btrfs_submit_bio_done);
		goto err;
8383 8384 8385 8386 8387 8388 8389 8390
	} else if (write) {
		/*
		 * If we aren't doing async submit, calculate the csum of the
		 * bio now.
		 */
		ret = btrfs_csum_one_bio(root, inode, bio, file_offset, 1);
		if (ret)
			goto err;
8391
	} else {
8392 8393
		ret = btrfs_lookup_and_bind_dio_csum(root, inode, dip, bio,
						     file_offset);
8394 8395 8396
		if (ret)
			goto err;
	}
8397
map:
8398
	ret = btrfs_map_bio(root, bio, 0, async_submit);
8399 8400 8401 8402 8403
err:
	bio_put(bio);
	return ret;
}

8404
static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
8405 8406 8407 8408 8409 8410 8411
				    int skip_sum)
{
	struct inode *inode = dip->inode;
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct bio *bio;
	struct bio *orig_bio = dip->orig_bio;
	struct bio_vec *bvec = orig_bio->bi_io_vec;
8412
	u64 start_sector = orig_bio->bi_iter.bi_sector;
8413 8414 8415
	u64 file_offset = dip->logical_offset;
	u64 submit_len = 0;
	u64 map_length;
8416
	u32 blocksize = root->sectorsize;
8417
	int async_submit = 0;
8418 8419 8420
	int nr_sectors;
	int ret;
	int i;
8421

8422
	map_length = orig_bio->bi_iter.bi_size;
8423 8424
	ret = btrfs_map_block(root->fs_info, bio_op(orig_bio),
			      start_sector << 9, &map_length, NULL, 0);
8425
	if (ret)
8426
		return -EIO;
8427

8428
	if (map_length >= orig_bio->bi_iter.bi_size) {
8429
		bio = orig_bio;
8430
		dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8431 8432 8433
		goto submit;
	}

David Woodhouse's avatar
David Woodhouse committed
8434
	/* async crcs make it difficult to collect full stripe writes. */
8435
	if (btrfs_get_alloc_profile(root, 1) & BTRFS_BLOCK_GROUP_RAID56_MASK)
David Woodhouse's avatar
David Woodhouse committed
8436 8437 8438 8439
		async_submit = 0;
	else
		async_submit = 1;

8440 8441 8442
	bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev, start_sector, GFP_NOFS);
	if (!bio)
		return -ENOMEM;
8443

8444
	bio_set_op_attrs(bio, bio_op(orig_bio), bio_flags(orig_bio));
8445 8446
	bio->bi_private = dip;
	bio->bi_end_io = btrfs_end_dio_bio;
8447
	btrfs_io_bio(bio)->logical = file_offset;
8448 8449
	atomic_inc(&dip->pending_bios);

8450
	while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt - 1)) {
8451 8452 8453 8454 8455 8456
		nr_sectors = BTRFS_BYTES_TO_BLKS(root->fs_info, bvec->bv_len);
		i = 0;
next_block:
		if (unlikely(map_length < submit_len + blocksize ||
		    bio_add_page(bio, bvec->bv_page, blocksize,
			    bvec->bv_offset + (i * blocksize)) < blocksize)) {
8457 8458 8459 8460 8461 8462 8463
			/*
			 * inc the count before we submit the bio so
			 * we know the end IO handler won't happen before
			 * we inc the count. Otherwise, the dip might get freed
			 * before we're done setting it up
			 */
			atomic_inc(&dip->pending_bios);
8464
			ret = __btrfs_submit_dio_bio(bio, inode,
8465
						     file_offset, skip_sum,
8466
						     async_submit);
8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481
			if (ret) {
				bio_put(bio);
				atomic_dec(&dip->pending_bios);
				goto out_err;
			}

			start_sector += submit_len >> 9;
			file_offset += submit_len;

			submit_len = 0;

			bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev,
						  start_sector, GFP_NOFS);
			if (!bio)
				goto out_err;
8482 8483
			bio_set_op_attrs(bio, bio_op(orig_bio),
					 bio_flags(orig_bio));
8484 8485
			bio->bi_private = dip;
			bio->bi_end_io = btrfs_end_dio_bio;
8486
			btrfs_io_bio(bio)->logical = file_offset;
8487

8488
			map_length = orig_bio->bi_iter.bi_size;
8489
			ret = btrfs_map_block(root->fs_info, bio_op(orig_bio),
8490
					      start_sector << 9,
8491 8492 8493 8494 8495
					      &map_length, NULL, 0);
			if (ret) {
				bio_put(bio);
				goto out_err;
			}
8496 8497

			goto next_block;
8498
		} else {
8499 8500 8501 8502 8503
			submit_len += blocksize;
			if (--nr_sectors) {
				i++;
				goto next_block;
			}
8504 8505 8506 8507
			bvec++;
		}
	}

8508
submit:
8509
	ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
8510
				     async_submit);
8511 8512 8513 8514 8515 8516 8517 8518 8519 8520
	if (!ret)
		return 0;

	bio_put(bio);
out_err:
	dip->errors = 1;
	/*
	 * before atomic variable goto zero, we must
	 * make sure dip->errors is perceived to be set.
	 */
8521
	smp_mb__before_atomic();
8522 8523 8524 8525 8526 8527 8528
	if (atomic_dec_and_test(&dip->pending_bios))
		bio_io_error(dip->orig_bio);

	/* bio_end_io() will handle error, so we needn't return it */
	return 0;
}

8529 8530
static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
				loff_t file_offset)
8531
{
8532 8533
	struct btrfs_dio_private *dip = NULL;
	struct bio *io_bio = NULL;
8534
	struct btrfs_io_bio *btrfs_bio;
8535
	int skip_sum;
8536
	bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8537 8538 8539 8540
	int ret = 0;

	skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;

8541 8542 8543 8544 8545 8546
	io_bio = btrfs_bio_clone(dio_bio, GFP_NOFS);
	if (!io_bio) {
		ret = -ENOMEM;
		goto free_ordered;
	}

8547
	dip = kzalloc(sizeof(*dip), GFP_NOFS);
8548 8549
	if (!dip) {
		ret = -ENOMEM;
8550
		goto free_ordered;
8551 8552
	}

8553
	dip->private = dio_bio->bi_private;
8554 8555
	dip->inode = inode;
	dip->logical_offset = file_offset;
8556 8557
	dip->bytes = dio_bio->bi_iter.bi_size;
	dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8558 8559 8560
	io_bio->bi_private = dip;
	dip->orig_bio = io_bio;
	dip->dio_bio = dio_bio;
8561
	atomic_set(&dip->pending_bios, 0);
8562 8563
	btrfs_bio = btrfs_io_bio(io_bio);
	btrfs_bio->logical = file_offset;
8564

8565
	if (write) {
8566
		io_bio->bi_end_io = btrfs_endio_direct_write;
8567
	} else {
8568
		io_bio->bi_end_io = btrfs_endio_direct_read;
8569 8570
		dip->subio_endio = btrfs_subio_endio_read;
	}
8571

8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586
	/*
	 * Reset the range for unsubmitted ordered extents (to a 0 length range)
	 * even if we fail to submit a bio, because in such case we do the
	 * corresponding error handling below and it must not be done a second
	 * time by btrfs_direct_IO().
	 */
	if (write) {
		struct btrfs_dio_data *dio_data = current->journal_info;

		dio_data->unsubmitted_oe_range_end = dip->logical_offset +
			dip->bytes;
		dio_data->unsubmitted_oe_range_start =
			dio_data->unsubmitted_oe_range_end;
	}

8587
	ret = btrfs_submit_direct_hook(dip, skip_sum);
8588
	if (!ret)
8589
		return;
8590

8591 8592
	if (btrfs_bio->end_io)
		btrfs_bio->end_io(btrfs_bio, ret);
8593

8594 8595
free_ordered:
	/*
8596 8597 8598 8599 8600 8601 8602
	 * If we arrived here it means either we failed to submit the dip
	 * or we either failed to clone the dio_bio or failed to allocate the
	 * dip. If we cloned the dio_bio and allocated the dip, we can just
	 * call bio_endio against our io_bio so that we get proper resource
	 * cleanup if we fail to submit the dip, otherwise, we must do the
	 * same as btrfs_endio_direct_[write|read] because we can't call these
	 * callbacks - they require an allocated dip and a clone of dio_bio.
8603
	 */
8604
	if (io_bio && dip) {
8605 8606
		io_bio->bi_error = -EIO;
		bio_endio(io_bio);
8607 8608 8609 8610 8611 8612 8613 8614
		/*
		 * The end io callbacks free our dip, do the final put on io_bio
		 * and all the cleanup and final put for dio_bio (through
		 * dio_end_io()).
		 */
		dip = NULL;
		io_bio = NULL;
	} else {
8615 8616 8617 8618 8619 8620
		if (write)
			btrfs_endio_direct_write_update_ordered(inode,
						file_offset,
						dio_bio->bi_iter.bi_size,
						0);
		else
8621 8622
			unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
			      file_offset + dio_bio->bi_iter.bi_size - 1);
8623

8624
		dio_bio->bi_error = -EIO;
8625 8626 8627 8628 8629
		/*
		 * Releases and cleans up our dio_bio, no need to bio_put()
		 * nor bio_endio()/bio_io_error() against dio_bio.
		 */
		dio_end_io(dio_bio, ret);
8630
	}
8631 8632 8633
	if (io_bio)
		bio_put(io_bio);
	kfree(dip);
8634 8635
}

8636
static ssize_t check_direct_IO(struct btrfs_root *root, struct kiocb *iocb,
8637
			const struct iov_iter *iter, loff_t offset)
Chris Mason's avatar
Chris Mason committed
8638 8639
{
	int seg;
8640
	int i;
Chris Mason's avatar
Chris Mason committed
8641 8642 8643 8644 8645 8646
	unsigned blocksize_mask = root->sectorsize - 1;
	ssize_t retval = -EINVAL;

	if (offset & blocksize_mask)
		goto out;

8647 8648
	if (iov_iter_alignment(iter) & blocksize_mask)
		goto out;
8649

8650
	/* If this is a write we don't need to check anymore */
8651
	if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8652 8653 8654 8655 8656 8657 8658 8659 8660
		return 0;
	/*
	 * Check to make sure we don't have duplicate iov_base's in this
	 * iovec, if so return EINVAL, otherwise we'll get csum errors
	 * when reading back.
	 */
	for (seg = 0; seg < iter->nr_segs; seg++) {
		for (i = seg + 1; i < iter->nr_segs; i++) {
			if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8661 8662
				goto out;
		}
Chris Mason's avatar
Chris Mason committed
8663 8664 8665 8666 8667
	}
	retval = 0;
out:
	return retval;
}
8668

8669
static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8670
{
8671 8672
	struct file *file = iocb->ki_filp;
	struct inode *inode = file->f_mapping->host;
8673 8674
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_dio_data dio_data = { 0 };
8675
	loff_t offset = iocb->ki_pos;
8676
	size_t count = 0;
8677
	int flags = 0;
8678 8679
	bool wakeup = true;
	bool relock = false;
8680
	ssize_t ret;
8681

8682
	if (check_direct_IO(BTRFS_I(inode)->root, iocb, iter, offset))
Chris Mason's avatar
Chris Mason committed
8683
		return 0;
8684

8685
	inode_dio_begin(inode);
8686
	smp_mb__after_atomic();
8687

8688
	/*
8689 8690 8691 8692
	 * The generic stuff only does filemap_write_and_wait_range, which
	 * isn't enough if we've written compressed pages to this area, so
	 * we need to flush the dirty pages again to make absolutely sure
	 * that any outstanding dirty pages are on disk.
8693
	 */
8694
	count = iov_iter_count(iter);
8695 8696
	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
		     &BTRFS_I(inode)->runtime_flags))
8697 8698
		filemap_fdatawrite_range(inode->i_mapping, offset,
					 offset + count - 1);
8699

8700
	if (iov_iter_rw(iter) == WRITE) {
8701 8702 8703 8704 8705 8706
		/*
		 * If the write DIO is beyond the EOF, we need update
		 * the isize, but it is protected by i_mutex. So we can
		 * not unlock the i_mutex at this case.
		 */
		if (offset + count <= inode->i_size) {
Al Viro's avatar
Al Viro committed
8707
			inode_unlock(inode);
8708 8709
			relock = true;
		}
8710
		ret = btrfs_delalloc_reserve_space(inode, offset, count);
8711
		if (ret)
8712
			goto out;
8713
		dio_data.outstanding_extents = div64_u64(count +
8714 8715 8716 8717 8718 8719 8720 8721
						BTRFS_MAX_EXTENT_SIZE - 1,
						BTRFS_MAX_EXTENT_SIZE);

		/*
		 * We need to know how many extents we reserved so that we can
		 * do the accounting properly if we go over the number we
		 * originally calculated.  Abuse current->journal_info for this.
		 */
8722
		dio_data.reserve = round_up(count, root->sectorsize);
8723 8724
		dio_data.unsubmitted_oe_range_start = (u64)offset;
		dio_data.unsubmitted_oe_range_end = (u64)offset;
8725
		current->journal_info = &dio_data;
8726
		down_read(&BTRFS_I(inode)->dio_sem);
8727 8728
	} else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
				     &BTRFS_I(inode)->runtime_flags)) {
8729
		inode_dio_end(inode);
8730 8731
		flags = DIO_LOCKING | DIO_SKIP_HOLES;
		wakeup = false;
8732 8733
	}

8734 8735
	ret = __blockdev_direct_IO(iocb, inode,
				   BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev,
8736
				   iter, btrfs_get_blocks_direct, NULL,
8737
				   btrfs_submit_direct, flags);
8738
	if (iov_iter_rw(iter) == WRITE) {
8739
		up_read(&BTRFS_I(inode)->dio_sem);
8740
		current->journal_info = NULL;
8741
		if (ret < 0 && ret != -EIOCBQUEUED) {
8742
			if (dio_data.reserve)
8743 8744
				btrfs_delalloc_release_space(inode, offset,
							     dio_data.reserve);
8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757
			/*
			 * On error we might have left some ordered extents
			 * without submitting corresponding bios for them, so
			 * cleanup them up to avoid other tasks getting them
			 * and waiting for them to complete forever.
			 */
			if (dio_data.unsubmitted_oe_range_start <
			    dio_data.unsubmitted_oe_range_end)
				btrfs_endio_direct_write_update_ordered(inode,
					dio_data.unsubmitted_oe_range_start,
					dio_data.unsubmitted_oe_range_end -
					dio_data.unsubmitted_oe_range_start,
					0);
8758
		} else if (ret >= 0 && (size_t)ret < count)
8759 8760
			btrfs_delalloc_release_space(inode, offset,
						     count - (size_t)ret);
8761
	}
8762
out:
8763
	if (wakeup)
8764
		inode_dio_end(inode);
8765
	if (relock)
Al Viro's avatar
Al Viro committed
8766
		inode_lock(inode);
8767 8768

	return ret;
8769 8770
}

8771 8772
#define BTRFS_FIEMAP_FLAGS	(FIEMAP_FLAG_SYNC)

Yehuda Sadeh's avatar
Yehuda Sadeh committed
8773 8774 8775
static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
		__u64 start, __u64 len)
{
8776 8777 8778 8779 8780 8781
	int	ret;

	ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
	if (ret)
		return ret;

8782
	return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
Yehuda Sadeh's avatar
Yehuda Sadeh committed
8783 8784
}

8785
int btrfs_readpage(struct file *file, struct page *page)
8786
{
8787 8788
	struct extent_io_tree *tree;
	tree = &BTRFS_I(page->mapping->host)->io_tree;
8789
	return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8790
}
8791

8792
static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
Chris Mason's avatar
Chris Mason committed
8793
{
8794
	struct extent_io_tree *tree;
8795 8796
	struct inode *inode = page->mapping->host;
	int ret;
8797 8798 8799 8800 8801 8802

	if (current->flags & PF_MEMALLOC) {
		redirty_page_for_writepage(wbc, page);
		unlock_page(page);
		return 0;
	}
8803 8804 8805 8806 8807 8808 8809 8810 8811 8812

	/*
	 * If we are under memory pressure we will call this directly from the
	 * VM, we need to make sure we have the inode referenced for the ordered
	 * extent.  If not just return like we didn't do anything.
	 */
	if (!igrab(inode)) {
		redirty_page_for_writepage(wbc, page);
		return AOP_WRITEPAGE_ACTIVATE;
	}
8813
	tree = &BTRFS_I(page->mapping->host)->io_tree;
8814 8815 8816
	ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc);
	btrfs_add_delayed_iput(inode);
	return ret;
8817 8818
}

8819 8820
static int btrfs_writepages(struct address_space *mapping,
			    struct writeback_control *wbc)
Chris Mason's avatar
Chris Mason committed
8821
{
8822
	struct extent_io_tree *tree;
8823

8824
	tree = &BTRFS_I(mapping->host)->io_tree;
Chris Mason's avatar
Chris Mason committed
8825 8826 8827
	return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
}

Chris Mason's avatar
Chris Mason committed
8828 8829 8830 8831
static int
btrfs_readpages(struct file *file, struct address_space *mapping,
		struct list_head *pages, unsigned nr_pages)
{
8832 8833
	struct extent_io_tree *tree;
	tree = &BTRFS_I(mapping->host)->io_tree;
Chris Mason's avatar
Chris Mason committed
8834 8835 8836
	return extent_readpages(tree, mapping, pages, nr_pages,
				btrfs_get_extent);
}
8837
static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8838
{
8839 8840
	struct extent_io_tree *tree;
	struct extent_map_tree *map;
8841
	int ret;
8842

8843 8844
	tree = &BTRFS_I(page->mapping->host)->io_tree;
	map = &BTRFS_I(page->mapping->host)->extent_tree;
8845
	ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8846 8847 8848
	if (ret == 1) {
		ClearPagePrivate(page);
		set_page_private(page, 0);
8849
		put_page(page);
Chris Mason's avatar
Chris Mason committed
8850
	}
8851
	return ret;
Chris Mason's avatar
Chris Mason committed
8852 8853
}

8854 8855
static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
{
8856 8857
	if (PageWriteback(page) || PageDirty(page))
		return 0;
8858
	return __btrfs_releasepage(page, gfp_flags & GFP_NOFS);
8859 8860
}

8861 8862
static void btrfs_invalidatepage(struct page *page, unsigned int offset,
				 unsigned int length)
Chris Mason's avatar
Chris Mason committed
8863
{
8864
	struct inode *inode = page->mapping->host;
8865
	struct extent_io_tree *tree;
8866
	struct btrfs_ordered_extent *ordered;
8867
	struct extent_state *cached_state = NULL;
8868
	u64 page_start = page_offset(page);
8869
	u64 page_end = page_start + PAGE_SIZE - 1;
8870 8871
	u64 start;
	u64 end;
8872
	int inode_evicting = inode->i_state & I_FREEING;
Chris Mason's avatar
Chris Mason committed
8873

8874 8875 8876 8877 8878 8879 8880
	/*
	 * we have the page locked, so new writeback can't start,
	 * and the dirty bit won't be cleared while we are here.
	 *
	 * Wait for IO on this page so that we can safely clear
	 * the PagePrivate2 bit and do ordered accounting
	 */
8881
	wait_on_page_writeback(page);
8882

8883
	tree = &BTRFS_I(inode)->io_tree;
8884 8885 8886 8887
	if (offset) {
		btrfs_releasepage(page, GFP_NOFS);
		return;
	}
8888 8889

	if (!inode_evicting)
8890
		lock_extent_bits(tree, page_start, page_end, &cached_state);
8891 8892 8893 8894
again:
	start = page_start;
	ordered = btrfs_lookup_ordered_range(inode, start,
					page_end - start + 1);
8895
	if (ordered) {
8896
		end = min(page_end, ordered->file_offset + ordered->len - 1);
8897 8898 8899 8900
		/*
		 * IO on this page will never be started, so we need
		 * to account for any ordered extents now
		 */
8901
		if (!inode_evicting)
8902
			clear_extent_bit(tree, start, end,
8903 8904 8905 8906
					 EXTENT_DIRTY | EXTENT_DELALLOC |
					 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
					 EXTENT_DEFRAG, 1, 0, &cached_state,
					 GFP_NOFS);
8907 8908 8909 8910
		/*
		 * whoever cleared the private bit is responsible
		 * for the finish_ordered_io
		 */
8911 8912 8913 8914 8915 8916 8917 8918
		if (TestClearPagePrivate2(page)) {
			struct btrfs_ordered_inode_tree *tree;
			u64 new_len;

			tree = &BTRFS_I(inode)->ordered_tree;

			spin_lock_irq(&tree->lock);
			set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
8919
			new_len = start - ordered->file_offset;
8920 8921 8922 8923 8924
			if (new_len < ordered->truncated_len)
				ordered->truncated_len = new_len;
			spin_unlock_irq(&tree->lock);

			if (btrfs_dec_test_ordered_pending(inode, &ordered,
8925 8926
							   start,
							   end - start + 1, 1))
8927
				btrfs_finish_ordered_io(ordered);
8928
		}
8929
		btrfs_put_ordered_extent(ordered);
8930 8931
		if (!inode_evicting) {
			cached_state = NULL;
8932
			lock_extent_bits(tree, start, end,
8933 8934
					 &cached_state);
		}
8935 8936 8937 8938

		start = end + 1;
		if (start < page_end)
			goto again;
8939 8940
	}

8941 8942 8943 8944 8945 8946 8947 8948 8949
	/*
	 * Qgroup reserved space handler
	 * Page here will be either
	 * 1) Already written to disk
	 *    In this case, its reserved space is released from data rsv map
	 *    and will be freed by delayed_ref handler finally.
	 *    So even we call qgroup_free_data(), it won't decrease reserved
	 *    space.
	 * 2) Not written to disk
8950 8951 8952 8953 8954
	 *    This means the reserved space should be freed here. However,
	 *    if a truncate invalidates the page (by clearing PageDirty)
	 *    and the page is accounted for while allocating extent
	 *    in btrfs_check_data_free_space() we let delayed_ref to
	 *    free the entire extent.
8955
	 */
8956 8957
	if (PageDirty(page))
		btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE);
8958 8959 8960 8961 8962 8963 8964 8965
	if (!inode_evicting) {
		clear_extent_bit(tree, page_start, page_end,
				 EXTENT_LOCKED | EXTENT_DIRTY |
				 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
				 EXTENT_DEFRAG, 1, 1,
				 &cached_state, GFP_NOFS);

		__btrfs_releasepage(page, GFP_NOFS);
8966 8967
	}

Chris Mason's avatar
Chris Mason committed
8968
	ClearPageChecked(page);
8969 8970 8971
	if (PagePrivate(page)) {
		ClearPagePrivate(page);
		set_page_private(page, 0);
8972
		put_page(page);
8973
	}
Chris Mason's avatar
Chris Mason committed
8974 8975
}

8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990
/*
 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
 * called from a page fault handler when a page is first dirtied. Hence we must
 * be careful to check for EOF conditions here. We set the page up correctly
 * for a written page which means we get ENOSPC checking when writing into
 * holes and correct delalloc and unwritten extent mapping on filesystems that
 * support these features.
 *
 * We are not allowed to take the i_mutex here so we have to play games to
 * protect against truncate races as the page could now be beyond EOF.  Because
 * vmtruncate() writes the inode size before removing pages, once we have the
 * page lock we can determine safely if the page is beyond EOF. If it is not
 * beyond EOF, then the page is guaranteed safe against truncation until we
 * unlock the page.
 */
8991
int btrfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
8992
{
8993
	struct page *page = vmf->page;
Al Viro's avatar
Al Viro committed
8994
	struct inode *inode = file_inode(vma->vm_file);
8995
	struct btrfs_root *root = BTRFS_I(inode)->root;
8996 8997
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
	struct btrfs_ordered_extent *ordered;
8998
	struct extent_state *cached_state = NULL;
8999 9000
	char *kaddr;
	unsigned long zero_start;
9001
	loff_t size;
9002
	int ret;
9003
	int reserved = 0;
9004
	u64 reserved_space;
9005
	u64 page_start;
9006
	u64 page_end;
9007 9008
	u64 end;

9009
	reserved_space = PAGE_SIZE;
9010

9011
	sb_start_pagefault(inode->i_sb);
9012
	page_start = page_offset(page);
9013
	page_end = page_start + PAGE_SIZE - 1;
9014
	end = page_end;
9015

9016 9017 9018 9019 9020 9021 9022 9023
	/*
	 * Reserving delalloc space after obtaining the page lock can lead to
	 * deadlock. For example, if a dirty page is locked by this function
	 * and the call to btrfs_delalloc_reserve_space() ends up triggering
	 * dirty page write out, then the btrfs_writepage() function could
	 * end up waiting indefinitely to get a lock on the page currently
	 * being processed by btrfs_page_mkwrite() function.
	 */
9024
	ret = btrfs_delalloc_reserve_space(inode, page_start,
9025
					   reserved_space);
9026
	if (!ret) {
9027
		ret = file_update_time(vma->vm_file);
9028 9029
		reserved = 1;
	}
9030 9031 9032 9033 9034
	if (ret) {
		if (ret == -ENOMEM)
			ret = VM_FAULT_OOM;
		else /* -ENOSPC, -EIO, etc */
			ret = VM_FAULT_SIGBUS;
9035 9036 9037
		if (reserved)
			goto out;
		goto out_noreserve;
9038
	}
9039

9040
	ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
9041
again:
9042 9043
	lock_page(page);
	size = i_size_read(inode);
9044

9045
	if ((page->mapping != inode->i_mapping) ||
9046
	    (page_start >= size)) {
9047 9048 9049
		/* page got truncated out from underneath us */
		goto out_unlock;
	}
9050 9051
	wait_on_page_writeback(page);

9052
	lock_extent_bits(io_tree, page_start, page_end, &cached_state);
9053 9054
	set_page_extent_mapped(page);

9055 9056 9057 9058
	/*
	 * we can't set the delalloc bits if there are pending ordered
	 * extents.  Drop our locks and wait for them to finish
	 */
9059
	ordered = btrfs_lookup_ordered_range(inode, page_start, page_end);
9060
	if (ordered) {
9061 9062
		unlock_extent_cached(io_tree, page_start, page_end,
				     &cached_state, GFP_NOFS);
9063
		unlock_page(page);
9064
		btrfs_start_ordered_extent(inode, ordered, 1);
9065 9066 9067 9068
		btrfs_put_ordered_extent(ordered);
		goto again;
	}

9069
	if (page->index == ((size - 1) >> PAGE_SHIFT)) {
9070
		reserved_space = round_up(size - page_start, root->sectorsize);
9071
		if (reserved_space < PAGE_SIZE) {
9072 9073 9074 9075 9076
			end = page_start + reserved_space - 1;
			spin_lock(&BTRFS_I(inode)->lock);
			BTRFS_I(inode)->outstanding_extents++;
			spin_unlock(&BTRFS_I(inode)->lock);
			btrfs_delalloc_release_space(inode, page_start,
9077
						PAGE_SIZE - reserved_space);
9078 9079 9080
		}
	}

Josef Bacik's avatar
Josef Bacik committed
9081 9082 9083 9084 9085 9086 9087
	/*
	 * XXX - page_mkwrite gets called every time the page is dirtied, even
	 * if it was already dirty, so for space accounting reasons we need to
	 * clear any delalloc bits for the range we are fixing to save.  There
	 * is probably a better way to do this, but for now keep consistent with
	 * prepare_pages in the normal write path.
	 */
9088
	clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
9089 9090
			  EXTENT_DIRTY | EXTENT_DELALLOC |
			  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
9091
			  0, 0, &cached_state, GFP_NOFS);
Josef Bacik's avatar
Josef Bacik committed
9092

9093
	ret = btrfs_set_extent_delalloc(inode, page_start, end,
9094
					&cached_state, 0);
Josef Bacik's avatar
Josef Bacik committed
9095
	if (ret) {
9096 9097
		unlock_extent_cached(io_tree, page_start, page_end,
				     &cached_state, GFP_NOFS);
Josef Bacik's avatar
Josef Bacik committed
9098 9099 9100
		ret = VM_FAULT_SIGBUS;
		goto out_unlock;
	}
9101
	ret = 0;
9102 9103

	/* page is wholly or partially inside EOF */
9104 9105
	if (page_start + PAGE_SIZE > size)
		zero_start = size & ~PAGE_MASK;
9106
	else
9107
		zero_start = PAGE_SIZE;
9108

9109
	if (zero_start != PAGE_SIZE) {
9110
		kaddr = kmap(page);
9111
		memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
9112 9113 9114
		flush_dcache_page(page);
		kunmap(page);
	}
9115
	ClearPageChecked(page);
9116
	set_page_dirty(page);
9117
	SetPageUptodate(page);
9118

9119 9120
	BTRFS_I(inode)->last_trans = root->fs_info->generation;
	BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
9121
	BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
9122

9123
	unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
9124 9125

out_unlock:
9126 9127
	if (!ret) {
		sb_end_pagefault(inode->i_sb);
9128
		return VM_FAULT_LOCKED;
9129
	}
9130
	unlock_page(page);
9131
out:
9132
	btrfs_delalloc_release_space(inode, page_start, reserved_space);
9133
out_noreserve:
9134
	sb_end_pagefault(inode->i_sb);
9135 9136 9137
	return ret;
}

9138
static int btrfs_truncate(struct inode *inode)
Chris Mason's avatar
Chris Mason committed
9139 9140
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
9141
	struct btrfs_block_rsv *rsv;
9142
	int ret = 0;
9143
	int err = 0;
Chris Mason's avatar
Chris Mason committed
9144
	struct btrfs_trans_handle *trans;
9145
	u64 mask = root->sectorsize - 1;
9146
	u64 min_size = btrfs_calc_trunc_metadata_size(root, 1);
Chris Mason's avatar
Chris Mason committed
9147

9148 9149 9150 9151
	ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
				       (u64)-1);
	if (ret)
		return ret;
Chris Mason's avatar
Chris Mason committed
9152

9153
	/*
9154
	 * Yes ladies and gentlemen, this is indeed ugly.  The fact is we have
9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167
	 * 3 things going on here
	 *
	 * 1) We need to reserve space for our orphan item and the space to
	 * delete our orphan item.  Lord knows we don't want to have a dangling
	 * orphan item because we didn't reserve space to remove it.
	 *
	 * 2) We need to reserve space to update our inode.
	 *
	 * 3) We need to have something to cache all the space that is going to
	 * be free'd up by the truncate operation, but also have some slack
	 * space reserved in case it uses space during the truncate (thank you
	 * very much snapshotting).
	 *
9168
	 * And we need these to all be separate.  The fact is we can use a lot of
9169
	 * space doing the truncate, and we have no earthly idea how much space
9170
	 * we will use, so we need the truncate reservation to be separate so it
9171 9172 9173 9174 9175
	 * doesn't end up using space reserved for updating the inode or
	 * removing the orphan item.  We also need to be able to stop the
	 * transaction and start a new one, which means we need to be able to
	 * update the inode several times, and we have no idea of knowing how
	 * many times that will be, so we can't just reserve 1 item for the
9176
	 * entirety of the operation, so that has to be done separately as well.
9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188
	 * Then there is the orphan item, which does indeed need to be held on
	 * to for the whole operation, and we need nobody to touch this reserved
	 * space except the orphan code.
	 *
	 * So that leaves us with
	 *
	 * 1) root->orphan_block_rsv - for the orphan deletion.
	 * 2) rsv - for the truncate reservation, which we will steal from the
	 * transaction reservation.
	 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
	 * updating the inode.
	 */
9189
	rsv = btrfs_alloc_block_rsv(root, BTRFS_BLOCK_RSV_TEMP);
9190 9191
	if (!rsv)
		return -ENOMEM;
9192
	rsv->size = min_size;
9193
	rsv->failfast = 1;
9194

9195
	/*
9196
	 * 1 for the truncate slack space
9197 9198
	 * 1 for updating the inode.
	 */
9199
	trans = btrfs_start_transaction(root, 2);
9200 9201 9202 9203
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out;
	}
9204

9205 9206
	/* Migrate the slack space for the truncate to our reserve */
	ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv, rsv,
9207
				      min_size, 0);
9208
	BUG_ON(ret);
9209

Josef Bacik's avatar
Josef Bacik committed
9210 9211 9212 9213 9214 9215 9216 9217
	/*
	 * So if we truncate and then write and fsync we normally would just
	 * write the extents that changed, which is a problem if we need to
	 * first truncate that entire inode.  So set this flag so we write out
	 * all of the extents in the inode to the sync log so we're completely
	 * safe.
	 */
	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9218
	trans->block_rsv = rsv;
9219

9220 9221 9222 9223
	while (1) {
		ret = btrfs_truncate_inode_items(trans, root, inode,
						 inode->i_size,
						 BTRFS_EXTENT_DATA_KEY);
9224
		if (ret != -ENOSPC && ret != -EAGAIN) {
9225
			err = ret;
9226
			break;
9227
		}
Chris Mason's avatar
Chris Mason committed
9228

9229
		trans->block_rsv = &root->fs_info->trans_block_rsv;
9230
		ret = btrfs_update_inode(trans, root, inode);
9231 9232 9233 9234
		if (ret) {
			err = ret;
			break;
		}
9235

9236
		btrfs_end_transaction(trans, root);
9237
		btrfs_btree_balance_dirty(root);
9238 9239 9240 9241 9242 9243 9244 9245 9246

		trans = btrfs_start_transaction(root, 2);
		if (IS_ERR(trans)) {
			ret = err = PTR_ERR(trans);
			trans = NULL;
			break;
		}

		ret = btrfs_block_rsv_migrate(&root->fs_info->trans_block_rsv,
9247
					      rsv, min_size, 0);
9248 9249
		BUG_ON(ret);	/* shouldn't happen */
		trans->block_rsv = rsv;
9250 9251 9252
	}

	if (ret == 0 && inode->i_nlink > 0) {
9253
		trans->block_rsv = root->orphan_block_rsv;
9254
		ret = btrfs_orphan_del(trans, inode);
9255 9256
		if (ret)
			err = ret;
9257 9258
	}

9259 9260 9261 9262 9263
	if (trans) {
		trans->block_rsv = &root->fs_info->trans_block_rsv;
		ret = btrfs_update_inode(trans, root, inode);
		if (ret && !err)
			err = ret;
9264

9265
		ret = btrfs_end_transaction(trans, root);
9266
		btrfs_btree_balance_dirty(root);
9267
	}
9268 9269 9270
out:
	btrfs_free_block_rsv(root, rsv);

9271 9272
	if (ret && !err)
		err = ret;
9273

9274
	return err;
Chris Mason's avatar
Chris Mason committed
9275 9276
}

9277 9278 9279
/*
 * create a new subvolume directory/inode (helper for the ioctl).
 */
9280
int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9281 9282 9283
			     struct btrfs_root *new_root,
			     struct btrfs_root *parent_root,
			     u64 new_dirid)
Chris Mason's avatar
Chris Mason committed
9284 9285
{
	struct inode *inode;
9286
	int err;
9287
	u64 index = 0;
Chris Mason's avatar
Chris Mason committed
9288

9289 9290 9291 9292
	inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
				new_dirid, new_dirid,
				S_IFDIR | (~current_umask() & S_IRWXUGO),
				&index);
9293
	if (IS_ERR(inode))
Christoph Hellwig's avatar
Christoph Hellwig committed
9294
		return PTR_ERR(inode);
Chris Mason's avatar
Chris Mason committed
9295 9296 9297
	inode->i_op = &btrfs_dir_inode_operations;
	inode->i_fop = &btrfs_dir_file_operations;

9298
	set_nlink(inode, 1);
9299
	btrfs_i_size_write(inode, 0);
9300
	unlock_new_inode(inode);
9301

9302 9303 9304
	err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
	if (err)
		btrfs_err(new_root->fs_info,
9305
			  "error inheriting subvolume %llu properties: %d",
9306 9307
			  new_root->root_key.objectid, err);

9308
	err = btrfs_update_inode(trans, new_root, inode);
9309

9310
	iput(inode);
9311
	return err;
Chris Mason's avatar
Chris Mason committed
9312 9313 9314 9315 9316
}

struct inode *btrfs_alloc_inode(struct super_block *sb)
{
	struct btrfs_inode *ei;
Yan, Zheng's avatar
Yan, Zheng committed
9317
	struct inode *inode;
Chris Mason's avatar
Chris Mason committed
9318 9319 9320 9321

	ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
	if (!ei)
		return NULL;
Yan, Zheng's avatar
Yan, Zheng committed
9322 9323 9324

	ei->root = NULL;
	ei->generation = 0;
9325
	ei->last_trans = 0;
9326
	ei->last_sub_trans = 0;
9327
	ei->logged_trans = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9328
	ei->delalloc_bytes = 0;
9329
	ei->defrag_bytes = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9330 9331
	ei->disk_i_size = 0;
	ei->flags = 0;
9332
	ei->csum_bytes = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9333
	ei->index_cnt = (u64)-1;
9334
	ei->dir_index = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9335
	ei->last_unlink_trans = 0;
9336
	ei->last_log_commit = 0;
9337
	ei->delayed_iput_count = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9338

9339 9340 9341
	spin_lock_init(&ei->lock);
	ei->outstanding_extents = 0;
	ei->reserved_extents = 0;
Yan, Zheng's avatar
Yan, Zheng committed
9342

9343
	ei->runtime_flags = 0;
9344
	ei->force_compress = BTRFS_COMPRESS_NONE;
Yan, Zheng's avatar
Yan, Zheng committed
9345

9346 9347
	ei->delayed_node = NULL;

9348 9349 9350
	ei->i_otime.tv_sec = 0;
	ei->i_otime.tv_nsec = 0;

Yan, Zheng's avatar
Yan, Zheng committed
9351
	inode = &ei->vfs_inode;
9352
	extent_map_tree_init(&ei->extent_tree);
9353 9354
	extent_io_tree_init(&ei->io_tree, &inode->i_data);
	extent_io_tree_init(&ei->io_failure_tree, &inode->i_data);
9355 9356
	ei->io_tree.track_uptodate = 1;
	ei->io_failure_tree.track_uptodate = 1;
9357
	atomic_set(&ei->sync_writers, 0);
Yan, Zheng's avatar
Yan, Zheng committed
9358
	mutex_init(&ei->log_mutex);
9359
	mutex_init(&ei->delalloc_mutex);
9360
	btrfs_ordered_inode_tree_init(&ei->ordered_tree);
Yan, Zheng's avatar
Yan, Zheng committed
9361
	INIT_LIST_HEAD(&ei->delalloc_inodes);
9362
	INIT_LIST_HEAD(&ei->delayed_iput);
Yan, Zheng's avatar
Yan, Zheng committed
9363
	RB_CLEAR_NODE(&ei->rb_node);
9364
	init_rwsem(&ei->dio_sem);
Yan, Zheng's avatar
Yan, Zheng committed
9365 9366

	return inode;
Chris Mason's avatar
Chris Mason committed
9367 9368
}

9369 9370 9371 9372 9373 9374 9375 9376
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
void btrfs_test_destroy_inode(struct inode *inode)
{
	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
#endif

Nick Piggin's avatar
Nick Piggin committed
9377 9378 9379 9380 9381 9382
static void btrfs_i_callback(struct rcu_head *head)
{
	struct inode *inode = container_of(head, struct inode, i_rcu);
	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}

Chris Mason's avatar
Chris Mason committed
9383 9384
void btrfs_destroy_inode(struct inode *inode)
{
9385
	struct btrfs_ordered_extent *ordered;
9386 9387
	struct btrfs_root *root = BTRFS_I(inode)->root;

9388
	WARN_ON(!hlist_empty(&inode->i_dentry));
Chris Mason's avatar
Chris Mason committed
9389
	WARN_ON(inode->i_data.nrpages);
9390 9391
	WARN_ON(BTRFS_I(inode)->outstanding_extents);
	WARN_ON(BTRFS_I(inode)->reserved_extents);
9392 9393
	WARN_ON(BTRFS_I(inode)->delalloc_bytes);
	WARN_ON(BTRFS_I(inode)->csum_bytes);
9394
	WARN_ON(BTRFS_I(inode)->defrag_bytes);
Chris Mason's avatar
Chris Mason committed
9395

9396 9397 9398 9399 9400 9401 9402 9403
	/*
	 * This can happen where we create an inode, but somebody else also
	 * created the same inode and we need to destroy the one we already
	 * created.
	 */
	if (!root)
		goto free;

9404 9405
	if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
		     &BTRFS_I(inode)->runtime_flags)) {
9406
		btrfs_info(root->fs_info, "inode %llu still on the orphan list",
9407
			btrfs_ino(inode));
9408
		atomic_dec(&root->orphan_inodes);
9409 9410
	}

9411
	while (1) {
9412 9413 9414 9415
		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
		if (!ordered)
			break;
		else {
9416 9417 9418
			btrfs_err(root->fs_info,
				  "found ordered extent %llu %llu on inode cleanup",
				  ordered->file_offset, ordered->len);
9419 9420 9421 9422 9423
			btrfs_remove_ordered_extent(inode, ordered);
			btrfs_put_ordered_extent(ordered);
			btrfs_put_ordered_extent(ordered);
		}
	}
9424
	btrfs_qgroup_check_reserved_leak(inode);
9425
	inode_tree_del(inode);
9426
	btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
9427
free:
Nick Piggin's avatar
Nick Piggin committed
9428
	call_rcu(&inode->i_rcu, btrfs_i_callback);
Chris Mason's avatar
Chris Mason committed
9429 9430
}

9431
int btrfs_drop_inode(struct inode *inode)
9432 9433
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
9434

9435 9436 9437
	if (root == NULL)
		return 1;

9438
	/* the snap/subvol tree is on deleting */
9439
	if (btrfs_root_refs(&root->root_item) == 0)
9440
		return 1;
9441
	else
9442
		return generic_drop_inode(inode);
9443 9444
}

9445
static void init_once(void *foo)
Chris Mason's avatar
Chris Mason committed
9446 9447 9448 9449 9450 9451 9452 9453
{
	struct btrfs_inode *ei = (struct btrfs_inode *) foo;

	inode_init_once(&ei->vfs_inode);
}

void btrfs_destroy_cachep(void)
{
9454 9455 9456 9457 9458
	/*
	 * Make sure all delayed rcu free inodes are flushed before we
	 * destroy cache.
	 */
	rcu_barrier();
9459 9460 9461 9462 9463
	kmem_cache_destroy(btrfs_inode_cachep);
	kmem_cache_destroy(btrfs_trans_handle_cachep);
	kmem_cache_destroy(btrfs_transaction_cachep);
	kmem_cache_destroy(btrfs_path_cachep);
	kmem_cache_destroy(btrfs_free_space_cachep);
Chris Mason's avatar
Chris Mason committed
9464 9465 9466 9467
}

int btrfs_init_cachep(void)
{
9468
	btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9469
			sizeof(struct btrfs_inode), 0,
9470 9471
			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
			init_once);
Chris Mason's avatar
Chris Mason committed
9472 9473
	if (!btrfs_inode_cachep)
		goto fail;
9474

9475
	btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9476
			sizeof(struct btrfs_trans_handle), 0,
9477
			SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
Chris Mason's avatar
Chris Mason committed
9478 9479
	if (!btrfs_trans_handle_cachep)
		goto fail;
9480

9481
	btrfs_transaction_cachep = kmem_cache_create("btrfs_transaction",
9482
			sizeof(struct btrfs_transaction), 0,
9483
			SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
Chris Mason's avatar
Chris Mason committed
9484 9485
	if (!btrfs_transaction_cachep)
		goto fail;
9486

9487
	btrfs_path_cachep = kmem_cache_create("btrfs_path",
9488
			sizeof(struct btrfs_path), 0,
9489
			SLAB_MEM_SPREAD, NULL);
Chris Mason's avatar
Chris Mason committed
9490 9491
	if (!btrfs_path_cachep)
		goto fail;
9492

9493
	btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9494
			sizeof(struct btrfs_free_space), 0,
9495
			SLAB_MEM_SPREAD, NULL);
9496 9497 9498
	if (!btrfs_free_space_cachep)
		goto fail;

Chris Mason's avatar
Chris Mason committed
9499 9500 9501 9502 9503 9504 9505 9506 9507
	return 0;
fail:
	btrfs_destroy_cachep();
	return -ENOMEM;
}

static int btrfs_getattr(struct vfsmount *mnt,
			 struct dentry *dentry, struct kstat *stat)
{
9508
	u64 delalloc_bytes;
9509
	struct inode *inode = d_inode(dentry);
9510 9511
	u32 blocksize = inode->i_sb->s_blocksize;

Chris Mason's avatar
Chris Mason committed
9512
	generic_fillattr(inode, stat);
9513
	stat->dev = BTRFS_I(inode)->root->anon_dev;
9514 9515 9516 9517

	spin_lock(&BTRFS_I(inode)->lock);
	delalloc_bytes = BTRFS_I(inode)->delalloc_bytes;
	spin_unlock(&BTRFS_I(inode)->lock);
9518
	stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9519
			ALIGN(delalloc_bytes, blocksize)) >> 9;
Chris Mason's avatar
Chris Mason committed
9520 9521 9522
	return 0;
}

9523 9524 9525 9526 9527 9528 9529 9530 9531 9532
static int btrfs_rename_exchange(struct inode *old_dir,
			      struct dentry *old_dentry,
			      struct inode *new_dir,
			      struct dentry *new_dentry)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(old_dir)->root;
	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
	struct inode *new_inode = new_dentry->d_inode;
	struct inode *old_inode = old_dentry->d_inode;
9533
	struct timespec ctime = current_time(old_inode);
9534 9535 9536 9537 9538 9539 9540
	struct dentry *parent;
	u64 old_ino = btrfs_ino(old_inode);
	u64 new_ino = btrfs_ino(new_inode);
	u64 old_idx = 0;
	u64 new_idx = 0;
	u64 root_objectid;
	int ret;
9541 9542
	bool root_log_pinned = false;
	bool dest_log_pinned = false;
9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586

	/* we only allow rename subvolume link between subvolumes */
	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
		return -EXDEV;

	/* close the race window with snapshot create/destroy ioctl */
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
		down_read(&root->fs_info->subvol_sem);
	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
		down_read(&dest->fs_info->subvol_sem);

	/*
	 * We want to reserve the absolute worst case amount of items.  So if
	 * both inodes are subvols and we need to unlink them then that would
	 * require 4 item modifications, but if they are both normal inodes it
	 * would require 5 item modifications, so we'll assume their normal
	 * inodes.  So 5 * 2 is 10, plus 2 for the new links, so 12 total items
	 * should cover the worst case number of items we'll modify.
	 */
	trans = btrfs_start_transaction(root, 12);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		goto out_notrans;
	}

	/*
	 * We need to find a free sequence number both in the source and
	 * in the destination directory for the exchange.
	 */
	ret = btrfs_set_inode_index(new_dir, &old_idx);
	if (ret)
		goto out_fail;
	ret = btrfs_set_inode_index(old_dir, &new_idx);
	if (ret)
		goto out_fail;

	BTRFS_I(old_inode)->dir_index = 0ULL;
	BTRFS_I(new_inode)->dir_index = 0ULL;

	/* Reference for the source. */
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
		/* force full log commit if subvolume involved. */
		btrfs_set_log_full_commit(root->fs_info, trans);
	} else {
9587 9588
		btrfs_pin_log_trans(root);
		root_log_pinned = true;
9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602
		ret = btrfs_insert_inode_ref(trans, dest,
					     new_dentry->d_name.name,
					     new_dentry->d_name.len,
					     old_ino,
					     btrfs_ino(new_dir), old_idx);
		if (ret)
			goto out_fail;
	}

	/* And now for the dest. */
	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
		/* force full log commit if subvolume involved. */
		btrfs_set_log_full_commit(dest->fs_info, trans);
	} else {
9603 9604
		btrfs_pin_log_trans(dest);
		dest_log_pinned = true;
9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644
		ret = btrfs_insert_inode_ref(trans, root,
					     old_dentry->d_name.name,
					     old_dentry->d_name.len,
					     new_ino,
					     btrfs_ino(old_dir), new_idx);
		if (ret)
			goto out_fail;
	}

	/* Update inode version and ctime/mtime. */
	inode_inc_iversion(old_dir);
	inode_inc_iversion(new_dir);
	inode_inc_iversion(old_inode);
	inode_inc_iversion(new_inode);
	old_dir->i_ctime = old_dir->i_mtime = ctime;
	new_dir->i_ctime = new_dir->i_mtime = ctime;
	old_inode->i_ctime = ctime;
	new_inode->i_ctime = ctime;

	if (old_dentry->d_parent != new_dentry->d_parent) {
		btrfs_record_unlink_dir(trans, old_dir, old_inode, 1);
		btrfs_record_unlink_dir(trans, new_dir, new_inode, 1);
	}

	/* src is a subvolume */
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
		root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
		ret = btrfs_unlink_subvol(trans, root, old_dir,
					  root_objectid,
					  old_dentry->d_name.name,
					  old_dentry->d_name.len);
	} else { /* src is an inode */
		ret = __btrfs_unlink_inode(trans, root, old_dir,
					   old_dentry->d_inode,
					   old_dentry->d_name.name,
					   old_dentry->d_name.len);
		if (!ret)
			ret = btrfs_update_inode(trans, root, old_inode);
	}
	if (ret) {
9645
		btrfs_abort_transaction(trans, ret);
9646 9647 9648 9649 9650 9651 9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662 9663 9664
		goto out_fail;
	}

	/* dest is a subvolume */
	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
		root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
		ret = btrfs_unlink_subvol(trans, dest, new_dir,
					  root_objectid,
					  new_dentry->d_name.name,
					  new_dentry->d_name.len);
	} else { /* dest is an inode */
		ret = __btrfs_unlink_inode(trans, dest, new_dir,
					   new_dentry->d_inode,
					   new_dentry->d_name.name,
					   new_dentry->d_name.len);
		if (!ret)
			ret = btrfs_update_inode(trans, dest, new_inode);
	}
	if (ret) {
9665
		btrfs_abort_transaction(trans, ret);
9666 9667 9668 9669 9670 9671 9672
		goto out_fail;
	}

	ret = btrfs_add_link(trans, new_dir, old_inode,
			     new_dentry->d_name.name,
			     new_dentry->d_name.len, 0, old_idx);
	if (ret) {
9673
		btrfs_abort_transaction(trans, ret);
9674 9675 9676 9677 9678 9679 9680
		goto out_fail;
	}

	ret = btrfs_add_link(trans, old_dir, new_inode,
			     old_dentry->d_name.name,
			     old_dentry->d_name.len, 0, new_idx);
	if (ret) {
9681
		btrfs_abort_transaction(trans, ret);
9682 9683 9684 9685 9686 9687 9688 9689
		goto out_fail;
	}

	if (old_inode->i_nlink == 1)
		BTRFS_I(old_inode)->dir_index = old_idx;
	if (new_inode->i_nlink == 1)
		BTRFS_I(new_inode)->dir_index = new_idx;

9690
	if (root_log_pinned) {
9691 9692 9693
		parent = new_dentry->d_parent;
		btrfs_log_new_name(trans, old_inode, old_dir, parent);
		btrfs_end_log_trans(root);
9694
		root_log_pinned = false;
9695
	}
9696
	if (dest_log_pinned) {
9697 9698 9699
		parent = old_dentry->d_parent;
		btrfs_log_new_name(trans, new_inode, new_dir, parent);
		btrfs_end_log_trans(dest);
9700
		dest_log_pinned = false;
9701 9702
	}
out_fail:
9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720 9721 9722 9723 9724 9725 9726 9727 9728 9729 9730
	/*
	 * If we have pinned a log and an error happened, we unpin tasks
	 * trying to sync the log and force them to fallback to a transaction
	 * commit if the log currently contains any of the inodes involved in
	 * this rename operation (to ensure we do not persist a log with an
	 * inconsistent state for any of these inodes or leading to any
	 * inconsistencies when replayed). If the transaction was aborted, the
	 * abortion reason is propagated to userspace when attempting to commit
	 * the transaction. If the log does not contain any of these inodes, we
	 * allow the tasks to sync it.
	 */
	if (ret && (root_log_pinned || dest_log_pinned)) {
		if (btrfs_inode_in_log(old_dir, root->fs_info->generation) ||
		    btrfs_inode_in_log(new_dir, root->fs_info->generation) ||
		    btrfs_inode_in_log(old_inode, root->fs_info->generation) ||
		    (new_inode &&
		     btrfs_inode_in_log(new_inode, root->fs_info->generation)))
		    btrfs_set_log_full_commit(root->fs_info, trans);

		if (root_log_pinned) {
			btrfs_end_log_trans(root);
			root_log_pinned = false;
		}
		if (dest_log_pinned) {
			btrfs_end_log_trans(dest);
			dest_log_pinned = false;
		}
	}
9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748 9749 9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774
	ret = btrfs_end_transaction(trans, root);
out_notrans:
	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
		up_read(&dest->fs_info->subvol_sem);
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
		up_read(&root->fs_info->subvol_sem);

	return ret;
}

static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct inode *dir,
				     struct dentry *dentry)
{
	int ret;
	struct inode *inode;
	u64 objectid;
	u64 index;

	ret = btrfs_find_free_ino(root, &objectid);
	if (ret)
		return ret;

	inode = btrfs_new_inode(trans, root, dir,
				dentry->d_name.name,
				dentry->d_name.len,
				btrfs_ino(dir),
				objectid,
				S_IFCHR | WHITEOUT_MODE,
				&index);

	if (IS_ERR(inode)) {
		ret = PTR_ERR(inode);
		return ret;
	}

	inode->i_op = &btrfs_special_inode_operations;
	init_special_inode(inode, inode->i_mode,
		WHITEOUT_DEV);

	ret = btrfs_init_inode_security(trans, inode, dir,
				&dentry->d_name);
	if (ret)
9775
		goto out;
9776 9777 9778 9779

	ret = btrfs_add_nondir(trans, dir, dentry,
				inode, 0, index);
	if (ret)
9780
		goto out;
9781 9782

	ret = btrfs_update_inode(trans, root, inode);
9783
out:
9784
	unlock_new_inode(inode);
9785 9786
	if (ret)
		inode_dec_link_count(inode);
9787 9788
	iput(inode);

9789
	return ret;
9790 9791
}

9792
static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9793 9794
			   struct inode *new_dir, struct dentry *new_dentry,
			   unsigned int flags)
Chris Mason's avatar
Chris Mason committed
9795 9796
{
	struct btrfs_trans_handle *trans;
9797
	unsigned int trans_num_items;
Chris Mason's avatar
Chris Mason committed
9798
	struct btrfs_root *root = BTRFS_I(old_dir)->root;
9799
	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9800 9801
	struct inode *new_inode = d_inode(new_dentry);
	struct inode *old_inode = d_inode(old_dentry);
9802
	u64 index = 0;
9803
	u64 root_objectid;
Chris Mason's avatar
Chris Mason committed
9804
	int ret;
9805
	u64 old_ino = btrfs_ino(old_inode);
9806
	bool log_pinned = false;
Chris Mason's avatar
Chris Mason committed
9807

9808
	if (btrfs_ino(new_dir) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9809 9810
		return -EPERM;

9811
	/* we only allow rename subvolume link between subvolumes */
9812
	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9813 9814
		return -EXDEV;

9815 9816
	if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
	    (new_inode && btrfs_ino(new_inode) == BTRFS_FIRST_FREE_OBJECTID))
Chris Mason's avatar
Chris Mason committed
9817
		return -ENOTEMPTY;
9818

9819 9820 9821
	if (S_ISDIR(old_inode->i_mode) && new_inode &&
	    new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
		return -ENOTEMPTY;
9822 9823 9824


	/* check for collisions, even if the  name isn't there */
9825
	ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9826 9827 9828 9829 9830 9831 9832
			     new_dentry->d_name.name,
			     new_dentry->d_name.len);

	if (ret) {
		if (ret == -EEXIST) {
			/* we shouldn't get
			 * eexist without a new_inode */
9833
			if (WARN_ON(!new_inode)) {
9834 9835 9836 9837 9838 9839 9840 9841 9842
				return ret;
			}
		} else {
			/* maybe -EOVERFLOW */
			return ret;
		}
	}
	ret = 0;

9843
	/*
9844 9845
	 * we're using rename to replace one file with another.  Start IO on it
	 * now so  we don't add too much work to the end of the transaction
9846
	 */
9847
	if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9848 9849
		filemap_flush(old_inode->i_mapping);

9850
	/* close the racy window with snapshot create/destroy ioctl */
9851
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9852
		down_read(&root->fs_info->subvol_sem);
9853 9854 9855 9856
	/*
	 * We want to reserve the absolute worst case amount of items.  So if
	 * both inodes are subvols and we need to unlink them then that would
	 * require 4 item modifications, but if they are both normal inodes it
9857
	 * would require 5 item modifications, so we'll assume they are normal
9858 9859
	 * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
	 * should cover the worst case number of items we'll modify.
9860 9861 9862
	 * If our rename has the whiteout flag, we need more 5 units for the
	 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
	 * when selinux is enabled).
9863
	 */
9864 9865 9866 9867
	trans_num_items = 11;
	if (flags & RENAME_WHITEOUT)
		trans_num_items += 5;
	trans = btrfs_start_transaction(root, trans_num_items);
9868
	if (IS_ERR(trans)) {
9869 9870 9871
		ret = PTR_ERR(trans);
		goto out_notrans;
	}
9872

9873 9874
	if (dest != root)
		btrfs_record_root_in_trans(trans, dest);
9875

9876 9877 9878
	ret = btrfs_set_inode_index(new_dir, &index);
	if (ret)
		goto out_fail;
9879

9880
	BTRFS_I(old_inode)->dir_index = 0ULL;
9881
	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9882
		/* force full log commit if subvolume involved. */
9883
		btrfs_set_log_full_commit(root->fs_info, trans);
9884
	} else {
9885 9886
		btrfs_pin_log_trans(root);
		log_pinned = true;
9887 9888 9889
		ret = btrfs_insert_inode_ref(trans, dest,
					     new_dentry->d_name.name,
					     new_dentry->d_name.len,
9890 9891
					     old_ino,
					     btrfs_ino(new_dir), index);
9892 9893
		if (ret)
			goto out_fail;
9894
	}
9895

9896 9897 9898
	inode_inc_iversion(old_dir);
	inode_inc_iversion(new_dir);
	inode_inc_iversion(old_inode);
9899 9900
	old_dir->i_ctime = old_dir->i_mtime =
	new_dir->i_ctime = new_dir->i_mtime =
9901
	old_inode->i_ctime = current_time(old_dir);
9902

9903 9904 9905
	if (old_dentry->d_parent != new_dentry->d_parent)
		btrfs_record_unlink_dir(trans, old_dir, old_inode, 1);

9906
	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9907 9908 9909 9910 9911
		root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
		ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
					old_dentry->d_name.name,
					old_dentry->d_name.len);
	} else {
9912
		ret = __btrfs_unlink_inode(trans, root, old_dir,
9913
					d_inode(old_dentry),
9914 9915 9916 9917
					old_dentry->d_name.name,
					old_dentry->d_name.len);
		if (!ret)
			ret = btrfs_update_inode(trans, root, old_inode);
9918
	}
9919
	if (ret) {
9920
		btrfs_abort_transaction(trans, ret);
9921 9922
		goto out_fail;
	}
Chris Mason's avatar
Chris Mason committed
9923 9924

	if (new_inode) {
9925
		inode_inc_iversion(new_inode);
9926
		new_inode->i_ctime = current_time(new_inode);
9927
		if (unlikely(btrfs_ino(new_inode) ==
9928 9929 9930 9931 9932 9933 9934 9935 9936
			     BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
			root_objectid = BTRFS_I(new_inode)->location.objectid;
			ret = btrfs_unlink_subvol(trans, dest, new_dir,
						root_objectid,
						new_dentry->d_name.name,
						new_dentry->d_name.len);
			BUG_ON(new_inode->i_nlink == 0);
		} else {
			ret = btrfs_unlink_inode(trans, dest, new_dir,
9937
						 d_inode(new_dentry),
9938 9939 9940
						 new_dentry->d_name.name,
						 new_dentry->d_name.len);
		}
9941
		if (!ret && new_inode->i_nlink == 0)
9942
			ret = btrfs_orphan_add(trans, d_inode(new_dentry));
9943
		if (ret) {
9944
			btrfs_abort_transaction(trans, ret);
9945 9946
			goto out_fail;
		}
Chris Mason's avatar
Chris Mason committed
9947
	}
9948

9949 9950
	ret = btrfs_add_link(trans, new_dir, old_inode,
			     new_dentry->d_name.name,
9951
			     new_dentry->d_name.len, 0, index);
9952
	if (ret) {
9953
		btrfs_abort_transaction(trans, ret);
9954 9955
		goto out_fail;
	}
Chris Mason's avatar
Chris Mason committed
9956

9957 9958 9959
	if (old_inode->i_nlink == 1)
		BTRFS_I(old_inode)->dir_index = index;

9960
	if (log_pinned) {
9961
		struct dentry *parent = new_dentry->d_parent;
9962

9963
		btrfs_log_new_name(trans, old_inode, old_dir, parent);
9964
		btrfs_end_log_trans(root);
9965
		log_pinned = false;
9966
	}
9967 9968 9969 9970 9971 9972

	if (flags & RENAME_WHITEOUT) {
		ret = btrfs_whiteout_for_rename(trans, root, old_dir,
						old_dentry);

		if (ret) {
9973
			btrfs_abort_transaction(trans, ret);
9974 9975
			goto out_fail;
		}
9976
	}
Chris Mason's avatar
Chris Mason committed
9977
out_fail:
9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997 9998 9999
	/*
	 * If we have pinned the log and an error happened, we unpin tasks
	 * trying to sync the log and force them to fallback to a transaction
	 * commit if the log currently contains any of the inodes involved in
	 * this rename operation (to ensure we do not persist a log with an
	 * inconsistent state for any of these inodes or leading to any
	 * inconsistencies when replayed). If the transaction was aborted, the
	 * abortion reason is propagated to userspace when attempting to commit
	 * the transaction. If the log does not contain any of these inodes, we
	 * allow the tasks to sync it.
	 */
	if (ret && log_pinned) {
		if (btrfs_inode_in_log(old_dir, root->fs_info->generation) ||
		    btrfs_inode_in_log(new_dir, root->fs_info->generation) ||
		    btrfs_inode_in_log(old_inode, root->fs_info->generation) ||
		    (new_inode &&
		     btrfs_inode_in_log(new_inode, root->fs_info->generation)))
		    btrfs_set_log_full_commit(root->fs_info, trans);

		btrfs_end_log_trans(root);
		log_pinned = false;
	}
10000
	btrfs_end_transaction(trans, root);
10001
out_notrans:
10002
	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
10003
		up_read(&root->fs_info->subvol_sem);
Josef Bacik's avatar
Josef Bacik committed
10004

Chris Mason's avatar
Chris Mason committed
10005 10006 10007
	return ret;
}

Miklos Szeredi's avatar
Miklos Szeredi committed
10008 10009 10010 10011
static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
			 struct inode *new_dir, struct dentry *new_dentry,
			 unsigned int flags)
{
10012
	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
Miklos Szeredi's avatar
Miklos Szeredi committed
10013 10014
		return -EINVAL;

10015 10016 10017 10018 10019
	if (flags & RENAME_EXCHANGE)
		return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
					  new_dentry);

	return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
Miklos Szeredi's avatar
Miklos Szeredi committed
10020 10021
}

10022 10023 10024
static void btrfs_run_delalloc_work(struct btrfs_work *work)
{
	struct btrfs_delalloc_work *delalloc_work;
10025
	struct inode *inode;
10026 10027 10028

	delalloc_work = container_of(work, struct btrfs_delalloc_work,
				     work);
10029
	inode = delalloc_work->inode;
10030 10031 10032
	filemap_flush(inode->i_mapping);
	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
				&BTRFS_I(inode)->runtime_flags))
10033
		filemap_flush(inode->i_mapping);
10034 10035

	if (delalloc_work->delay_iput)
10036
		btrfs_add_delayed_iput(inode);
10037
	else
10038
		iput(inode);
10039 10040 10041 10042
	complete(&delalloc_work->completion);
}

struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
10043
						    int delay_iput)
10044 10045 10046
{
	struct btrfs_delalloc_work *work;

10047
	work = kmalloc(sizeof(*work), GFP_NOFS);
10048 10049 10050 10051 10052 10053 10054
	if (!work)
		return NULL;

	init_completion(&work->completion);
	INIT_LIST_HEAD(&work->list);
	work->inode = inode;
	work->delay_iput = delay_iput;
10055 10056 10057
	WARN_ON_ONCE(!inode);
	btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
			btrfs_run_delalloc_work, NULL, NULL);
10058 10059 10060 10061 10062 10063 10064

	return work;
}

void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
{
	wait_for_completion(&work->completion);
10065
	kfree(work);
10066 10067
}

10068 10069 10070 10071
/*
 * some fairly slow code that needs optimization. This walks the list
 * of all the inodes with pending delalloc and forces them to disk.
 */
10072 10073
static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
				   int nr)
10074 10075
{
	struct btrfs_inode *binode;
10076
	struct inode *inode;
10077 10078
	struct btrfs_delalloc_work *work, *next;
	struct list_head works;
10079
	struct list_head splice;
10080
	int ret = 0;
10081

10082
	INIT_LIST_HEAD(&works);
10083
	INIT_LIST_HEAD(&splice);
10084

10085
	mutex_lock(&root->delalloc_mutex);
10086 10087
	spin_lock(&root->delalloc_lock);
	list_splice_init(&root->delalloc_inodes, &splice);
10088 10089
	while (!list_empty(&splice)) {
		binode = list_entry(splice.next, struct btrfs_inode,
10090
				    delalloc_inodes);
10091

10092 10093
		list_move_tail(&binode->delalloc_inodes,
			       &root->delalloc_inodes);
10094
		inode = igrab(&binode->vfs_inode);
10095
		if (!inode) {
10096
			cond_resched_lock(&root->delalloc_lock);
10097
			continue;
10098
		}
10099
		spin_unlock(&root->delalloc_lock);
10100

10101
		work = btrfs_alloc_delalloc_work(inode, delay_iput);
10102
		if (!work) {
10103 10104 10105 10106
			if (delay_iput)
				btrfs_add_delayed_iput(inode);
			else
				iput(inode);
10107
			ret = -ENOMEM;
10108
			goto out;
10109
		}
10110
		list_add_tail(&work->list, &works);
10111 10112
		btrfs_queue_work(root->fs_info->flush_workers,
				 &work->work);
10113 10114
		ret++;
		if (nr != -1 && ret >= nr)
10115
			goto out;
10116
		cond_resched();
10117
		spin_lock(&root->delalloc_lock);
10118
	}
10119
	spin_unlock(&root->delalloc_lock);
10120

10121
out:
10122 10123 10124 10125 10126 10127 10128 10129 10130 10131
	list_for_each_entry_safe(work, next, &works, list) {
		list_del_init(&work->list);
		btrfs_wait_and_free_delalloc_work(work);
	}

	if (!list_empty_careful(&splice)) {
		spin_lock(&root->delalloc_lock);
		list_splice_tail(&splice, &root->delalloc_inodes);
		spin_unlock(&root->delalloc_lock);
	}
10132
	mutex_unlock(&root->delalloc_mutex);
10133 10134
	return ret;
}
10135

10136 10137 10138
int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
{
	int ret;
10139

10140
	if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
10141 10142
		return -EROFS;

10143 10144 10145
	ret = __start_delalloc_inodes(root, delay_iput, -1);
	if (ret > 0)
		ret = 0;
10146 10147
	/*
	 * the filemap_flush will queue IO into the worker threads, but
10148 10149 10150 10151
	 * we have to make sure the IO is actually started and that
	 * ordered extents get created before we return
	 */
	atomic_inc(&root->fs_info->async_submit_draining);
10152
	while (atomic_read(&root->fs_info->nr_async_submits) ||
10153
	      atomic_read(&root->fs_info->async_delalloc_pages)) {
10154
		wait_event(root->fs_info->async_submit_wait,
10155 10156
		   (atomic_read(&root->fs_info->nr_async_submits) == 0 &&
		    atomic_read(&root->fs_info->async_delalloc_pages) == 0));
10157 10158
	}
	atomic_dec(&root->fs_info->async_submit_draining);
10159 10160 10161
	return ret;
}

10162 10163
int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
			       int nr)
10164 10165 10166 10167 10168
{
	struct btrfs_root *root;
	struct list_head splice;
	int ret;

10169
	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10170 10171 10172 10173
		return -EROFS;

	INIT_LIST_HEAD(&splice);

10174
	mutex_lock(&fs_info->delalloc_root_mutex);
10175 10176
	spin_lock(&fs_info->delalloc_root_lock);
	list_splice_init(&fs_info->delalloc_roots, &splice);
10177
	while (!list_empty(&splice) && nr) {
10178 10179 10180 10181 10182 10183 10184 10185
		root = list_first_entry(&splice, struct btrfs_root,
					delalloc_root);
		root = btrfs_grab_fs_root(root);
		BUG_ON(!root);
		list_move_tail(&root->delalloc_root,
			       &fs_info->delalloc_roots);
		spin_unlock(&fs_info->delalloc_root_lock);

10186
		ret = __start_delalloc_inodes(root, delay_iput, nr);
10187
		btrfs_put_fs_root(root);
10188
		if (ret < 0)
10189 10190
			goto out;

10191 10192 10193 10194
		if (nr != -1) {
			nr -= ret;
			WARN_ON(nr < 0);
		}
10195
		spin_lock(&fs_info->delalloc_root_lock);
10196
	}
10197
	spin_unlock(&fs_info->delalloc_root_lock);
10198

10199
	ret = 0;
10200 10201 10202 10203 10204 10205 10206 10207 10208
	atomic_inc(&fs_info->async_submit_draining);
	while (atomic_read(&fs_info->nr_async_submits) ||
	      atomic_read(&fs_info->async_delalloc_pages)) {
		wait_event(fs_info->async_submit_wait,
		   (atomic_read(&fs_info->nr_async_submits) == 0 &&
		    atomic_read(&fs_info->async_delalloc_pages) == 0));
	}
	atomic_dec(&fs_info->async_submit_draining);
out:
10209
	if (!list_empty_careful(&splice)) {
10210 10211 10212
		spin_lock(&fs_info->delalloc_root_lock);
		list_splice_tail(&splice, &fs_info->delalloc_roots);
		spin_unlock(&fs_info->delalloc_root_lock);
10213
	}
10214
	mutex_unlock(&fs_info->delalloc_root_mutex);
10215
	return ret;
10216 10217
}

Chris Mason's avatar
Chris Mason committed
10218 10219 10220 10221 10222 10223 10224
static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
			 const char *symname)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(dir)->root;
	struct btrfs_path *path;
	struct btrfs_key key;
10225
	struct inode *inode = NULL;
Chris Mason's avatar
Chris Mason committed
10226 10227 10228
	int err;
	int drop_inode = 0;
	u64 objectid;
10229
	u64 index = 0;
Chris Mason's avatar
Chris Mason committed
10230 10231
	int name_len;
	int datasize;
10232
	unsigned long ptr;
Chris Mason's avatar
Chris Mason committed
10233
	struct btrfs_file_extent_item *ei;
10234
	struct extent_buffer *leaf;
Chris Mason's avatar
Chris Mason committed
10235

10236
	name_len = strlen(symname);
Chris Mason's avatar
Chris Mason committed
10237 10238
	if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(root))
		return -ENAMETOOLONG;
10239

Josef Bacik's avatar
Josef Bacik committed
10240 10241 10242
	/*
	 * 2 items for inode item and ref
	 * 2 items for dir items
10243 10244
	 * 1 item for updating parent inode item
	 * 1 item for the inline extent item
Josef Bacik's avatar
Josef Bacik committed
10245 10246
	 * 1 item for xattr if selinux is on
	 */
10247
	trans = btrfs_start_transaction(root, 7);
10248 10249
	if (IS_ERR(trans))
		return PTR_ERR(trans);
10250

10251 10252 10253 10254
	err = btrfs_find_free_ino(root, &objectid);
	if (err)
		goto out_unlock;

10255
	inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10256
				dentry->d_name.len, btrfs_ino(dir), objectid,
10257
				S_IFLNK|S_IRWXUGO, &index);
10258 10259
	if (IS_ERR(inode)) {
		err = PTR_ERR(inode);
Chris Mason's avatar
Chris Mason committed
10260
		goto out_unlock;
10261
	}
Chris Mason's avatar
Chris Mason committed
10262

10263 10264 10265 10266 10267 10268 10269 10270
	/*
	* If the active LSM wants to access the inode during
	* d_instantiate it needs these. Smack checks to see
	* if the filesystem supports xattrs by looking at the
	* ops vector.
	*/
	inode->i_fop = &btrfs_file_operations;
	inode->i_op = &btrfs_file_inode_operations;
10271 10272 10273 10274 10275 10276
	inode->i_mapping->a_ops = &btrfs_aops;
	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;

	err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
	if (err)
		goto out_unlock_inode;
10277

Chris Mason's avatar
Chris Mason committed
10278
	path = btrfs_alloc_path();
10279 10280
	if (!path) {
		err = -ENOMEM;
10281
		goto out_unlock_inode;
10282
	}
10283
	key.objectid = btrfs_ino(inode);
Chris Mason's avatar
Chris Mason committed
10284
	key.offset = 0;
10285
	key.type = BTRFS_EXTENT_DATA_KEY;
Chris Mason's avatar
Chris Mason committed
10286 10287 10288
	datasize = btrfs_file_extent_calc_inline_size(name_len);
	err = btrfs_insert_empty_item(trans, root, path, &key,
				      datasize);
10289
	if (err) {
10290
		btrfs_free_path(path);
10291
		goto out_unlock_inode;
10292
	}
10293 10294 10295 10296 10297
	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0],
			    struct btrfs_file_extent_item);
	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
	btrfs_set_file_extent_type(leaf, ei,
Chris Mason's avatar
Chris Mason committed
10298
				   BTRFS_FILE_EXTENT_INLINE);
10299 10300 10301 10302 10303
	btrfs_set_file_extent_encryption(leaf, ei, 0);
	btrfs_set_file_extent_compression(leaf, ei, 0);
	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
	btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);

Chris Mason's avatar
Chris Mason committed
10304
	ptr = btrfs_file_extent_inline_start(ei);
10305 10306
	write_extent_buffer(leaf, symname, ptr, name_len);
	btrfs_mark_buffer_dirty(leaf);
Chris Mason's avatar
Chris Mason committed
10307
	btrfs_free_path(path);
10308

Chris Mason's avatar
Chris Mason committed
10309
	inode->i_op = &btrfs_symlink_inode_operations;
10310
	inode_nohighmem(inode);
Chris Mason's avatar
Chris Mason committed
10311
	inode->i_mapping->a_ops = &btrfs_symlink_aops;
Yan Zheng's avatar
Yan Zheng committed
10312
	inode_set_bytes(inode, name_len);
10313
	btrfs_i_size_write(inode, name_len);
10314
	err = btrfs_update_inode(trans, root, inode);
10315 10316 10317 10318 10319 10320 10321
	/*
	 * Last step, add directory indexes for our symlink inode. This is the
	 * last step to avoid extra cleanup of these indexes if an error happens
	 * elsewhere above.
	 */
	if (!err)
		err = btrfs_add_nondir(trans, dir, dentry, inode, 0, index);
10322
	if (err) {
10323
		drop_inode = 1;
10324 10325 10326 10327 10328
		goto out_unlock_inode;
	}

	unlock_new_inode(inode);
	d_instantiate(dentry, inode);
Chris Mason's avatar
Chris Mason committed
10329 10330

out_unlock:
10331
	btrfs_end_transaction(trans, root);
Chris Mason's avatar
Chris Mason committed
10332 10333 10334 10335
	if (drop_inode) {
		inode_dec_link_count(inode);
		iput(inode);
	}
10336
	btrfs_btree_balance_dirty(root);
Chris Mason's avatar
Chris Mason committed
10337
	return err;
10338 10339 10340 10341 10342

out_unlock_inode:
	drop_inode = 1;
	unlock_new_inode(inode);
	goto out_unlock;
Chris Mason's avatar
Chris Mason committed
10343
}
10344

10345 10346 10347 10348
static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
				       u64 start, u64 num_bytes, u64 min_size,
				       loff_t actual_len, u64 *alloc_hint,
				       struct btrfs_trans_handle *trans)
Yan Zheng's avatar
Yan Zheng committed
10349
{
Josef Bacik's avatar
Josef Bacik committed
10350 10351
	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
	struct extent_map *em;
Yan Zheng's avatar
Yan Zheng committed
10352 10353 10354
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_key ins;
	u64 cur_offset = start;
10355
	u64 i_size;
10356
	u64 cur_bytes;
10357
	u64 last_alloc = (u64)-1;
Yan Zheng's avatar
Yan Zheng committed
10358
	int ret = 0;
10359
	bool own_trans = true;
10360
	u64 end = start + num_bytes - 1;
Yan Zheng's avatar
Yan Zheng committed
10361

10362 10363
	if (trans)
		own_trans = false;
Yan Zheng's avatar
Yan Zheng committed
10364
	while (num_bytes > 0) {
10365 10366 10367 10368 10369 10370
		if (own_trans) {
			trans = btrfs_start_transaction(root, 3);
			if (IS_ERR(trans)) {
				ret = PTR_ERR(trans);
				break;
			}
10371 10372
		}

10373
		cur_bytes = min_t(u64, num_bytes, SZ_256M);
10374
		cur_bytes = max(cur_bytes, min_size);
10375 10376 10377 10378 10379 10380 10381
		/*
		 * If we are severely fragmented we could end up with really
		 * small allocations, so if the allocator is returning small
		 * chunks lets make its job easier by only searching for those
		 * sized chunks.
		 */
		cur_bytes = min(cur_bytes, last_alloc);
10382 10383
		ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
				min_size, 0, *alloc_hint, &ins, 1, 0);
10384
		if (ret) {
10385 10386
			if (own_trans)
				btrfs_end_transaction(trans, root);
10387
			break;
Yan Zheng's avatar
Yan Zheng committed
10388
		}
10389
		btrfs_dec_block_group_reservations(root->fs_info, ins.objectid);
10390

10391
		last_alloc = ins.offset;
Yan Zheng's avatar
Yan Zheng committed
10392 10393 10394
		ret = insert_reserved_file_extent(trans, inode,
						  cur_offset, ins.objectid,
						  ins.offset, ins.offset,
10395
						  ins.offset, 0, 0, 0,
Yan Zheng's avatar
Yan Zheng committed
10396
						  BTRFS_FILE_EXTENT_PREALLOC);
10397
		if (ret) {
10398
			btrfs_free_reserved_extent(root, ins.objectid,
10399
						   ins.offset, 0);
10400
			btrfs_abort_transaction(trans, ret);
10401 10402 10403 10404
			if (own_trans)
				btrfs_end_transaction(trans, root);
			break;
		}
10405

10406 10407
		btrfs_drop_extent_cache(inode, cur_offset,
					cur_offset + ins.offset -1, 0);
10408

Josef Bacik's avatar
Josef Bacik committed
10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420
		em = alloc_extent_map();
		if (!em) {
			set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
				&BTRFS_I(inode)->runtime_flags);
			goto next;
		}

		em->start = cur_offset;
		em->orig_start = cur_offset;
		em->len = ins.offset;
		em->block_start = ins.objectid;
		em->block_len = ins.offset;
10421
		em->orig_block_len = ins.offset;
Josef Bacik's avatar
Josef Bacik committed
10422
		em->ram_bytes = ins.offset;
Josef Bacik's avatar
Josef Bacik committed
10423 10424 10425 10426 10427 10428
		em->bdev = root->fs_info->fs_devices->latest_bdev;
		set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
		em->generation = trans->transid;

		while (1) {
			write_lock(&em_tree->lock);
Josef Bacik's avatar
Josef Bacik committed
10429
			ret = add_extent_mapping(em_tree, em, 1);
Josef Bacik's avatar
Josef Bacik committed
10430 10431 10432 10433 10434 10435 10436 10437 10438
			write_unlock(&em_tree->lock);
			if (ret != -EEXIST)
				break;
			btrfs_drop_extent_cache(inode, cur_offset,
						cur_offset + ins.offset - 1,
						0);
		}
		free_extent_map(em);
next:
Yan Zheng's avatar
Yan Zheng committed
10439 10440
		num_bytes -= ins.offset;
		cur_offset += ins.offset;
10441
		*alloc_hint = ins.objectid + ins.offset;
10442

10443
		inode_inc_iversion(inode);
10444
		inode->i_ctime = current_time(inode);
10445
		BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
Yan Zheng's avatar
Yan Zheng committed
10446
		if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10447 10448
		    (actual_len > inode->i_size) &&
		    (cur_offset > inode->i_size)) {
10449
			if (cur_offset > actual_len)
10450
				i_size = actual_len;
10451
			else
10452 10453 10454
				i_size = cur_offset;
			i_size_write(inode, i_size);
			btrfs_ordered_update_i_size(inode, i_size, NULL);
10455 10456
		}

Yan Zheng's avatar
Yan Zheng committed
10457
		ret = btrfs_update_inode(trans, root, inode);
10458 10459

		if (ret) {
10460
			btrfs_abort_transaction(trans, ret);
10461 10462 10463 10464
			if (own_trans)
				btrfs_end_transaction(trans, root);
			break;
		}
Yan Zheng's avatar
Yan Zheng committed
10465

10466 10467
		if (own_trans)
			btrfs_end_transaction(trans, root);
10468
	}
10469 10470 10471
	if (cur_offset < end)
		btrfs_free_reserved_data_space(inode, cur_offset,
			end - cur_offset + 1);
Yan Zheng's avatar
Yan Zheng committed
10472 10473 10474
	return ret;
}

10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492
int btrfs_prealloc_file_range(struct inode *inode, int mode,
			      u64 start, u64 num_bytes, u64 min_size,
			      loff_t actual_len, u64 *alloc_hint)
{
	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
					   min_size, actual_len, alloc_hint,
					   NULL);
}

int btrfs_prealloc_file_range_trans(struct inode *inode,
				    struct btrfs_trans_handle *trans, int mode,
				    u64 start, u64 num_bytes, u64 min_size,
				    loff_t actual_len, u64 *alloc_hint)
{
	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
					   min_size, actual_len, alloc_hint, trans);
}

10493 10494 10495 10496 10497
static int btrfs_set_page_dirty(struct page *page)
{
	return __set_page_dirty_nobuffers(page);
}

10498
static int btrfs_permission(struct inode *inode, int mask)
Yan's avatar
Yan committed
10499
{
10500
	struct btrfs_root *root = BTRFS_I(inode)->root;
10501
	umode_t mode = inode->i_mode;
10502

10503 10504 10505 10506 10507 10508 10509
	if (mask & MAY_WRITE &&
	    (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
		if (btrfs_root_readonly(root))
			return -EROFS;
		if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
			return -EACCES;
	}
10510
	return generic_permission(inode, mask);
Yan's avatar
Yan committed
10511
}
Chris Mason's avatar
Chris Mason committed
10512

10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546
static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = BTRFS_I(dir)->root;
	struct inode *inode = NULL;
	u64 objectid;
	u64 index;
	int ret = 0;

	/*
	 * 5 units required for adding orphan entry
	 */
	trans = btrfs_start_transaction(root, 5);
	if (IS_ERR(trans))
		return PTR_ERR(trans);

	ret = btrfs_find_free_ino(root, &objectid);
	if (ret)
		goto out;

	inode = btrfs_new_inode(trans, root, dir, NULL, 0,
				btrfs_ino(dir), objectid, mode, &index);
	if (IS_ERR(inode)) {
		ret = PTR_ERR(inode);
		inode = NULL;
		goto out;
	}

	inode->i_fop = &btrfs_file_operations;
	inode->i_op = &btrfs_file_inode_operations;

	inode->i_mapping->a_ops = &btrfs_aops;
	BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;

10547 10548 10549 10550 10551 10552 10553
	ret = btrfs_init_inode_security(trans, inode, dir, NULL);
	if (ret)
		goto out_inode;

	ret = btrfs_update_inode(trans, root, inode);
	if (ret)
		goto out_inode;
10554 10555
	ret = btrfs_orphan_add(trans, inode);
	if (ret)
10556
		goto out_inode;
10557

10558 10559 10560 10561 10562 10563 10564 10565
	/*
	 * We set number of links to 0 in btrfs_new_inode(), and here we set
	 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
	 * through:
	 *
	 *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
	 */
	set_nlink(inode, 1);
10566
	unlock_new_inode(inode);
10567 10568 10569 10570 10571 10572 10573 10574 10575 10576
	d_tmpfile(dentry, inode);
	mark_inode_dirty(inode);

out:
	btrfs_end_transaction(trans, root);
	if (ret)
		iput(inode);
	btrfs_balance_delayed_items(root);
	btrfs_btree_balance_dirty(root);
	return ret;
10577 10578 10579 10580 10581

out_inode:
	unlock_new_inode(inode);
	goto out;

10582 10583
}

10584
static const struct inode_operations btrfs_dir_inode_operations = {
10585
	.getattr	= btrfs_getattr,
Chris Mason's avatar
Chris Mason committed
10586 10587 10588 10589 10590 10591
	.lookup		= btrfs_lookup,
	.create		= btrfs_create,
	.unlink		= btrfs_unlink,
	.link		= btrfs_link,
	.mkdir		= btrfs_mkdir,
	.rmdir		= btrfs_rmdir,
10592
	.rename		= btrfs_rename2,
Chris Mason's avatar
Chris Mason committed
10593 10594
	.symlink	= btrfs_symlink,
	.setattr	= btrfs_setattr,
Josef Bacik's avatar
Josef Bacik committed
10595
	.mknod		= btrfs_mknod,
Josef Bacik's avatar
Josef Bacik committed
10596
	.listxattr	= btrfs_listxattr,
Yan's avatar
Yan committed
10597
	.permission	= btrfs_permission,
10598
	.get_acl	= btrfs_get_acl,
10599
	.set_acl	= btrfs_set_acl,
10600
	.update_time	= btrfs_update_time,
10601
	.tmpfile        = btrfs_tmpfile,
Chris Mason's avatar
Chris Mason committed
10602
};
10603
static const struct inode_operations btrfs_dir_ro_inode_operations = {
Chris Mason's avatar
Chris Mason committed
10604
	.lookup		= btrfs_lookup,
Yan's avatar
Yan committed
10605
	.permission	= btrfs_permission,
10606
	.update_time	= btrfs_update_time,
Chris Mason's avatar
Chris Mason committed
10607
};
10608

10609
static const struct file_operations btrfs_dir_file_operations = {
Chris Mason's avatar
Chris Mason committed
10610 10611
	.llseek		= generic_file_llseek,
	.read		= generic_read_dir,
10612
	.iterate_shared	= btrfs_real_readdir,
10613
	.unlocked_ioctl	= btrfs_ioctl,
Chris Mason's avatar
Chris Mason committed
10614
#ifdef CONFIG_COMPAT
10615
	.compat_ioctl	= btrfs_compat_ioctl,
Chris Mason's avatar
Chris Mason committed
10616
#endif
Sage Weil's avatar
Sage Weil committed
10617
	.release        = btrfs_release_file,
10618
	.fsync		= btrfs_sync_file,
Chris Mason's avatar
Chris Mason committed
10619 10620
};

10621
static const struct extent_io_ops btrfs_extent_io_ops = {
10622
	.fill_delalloc = run_delalloc_range,
10623
	.submit_bio_hook = btrfs_submit_bio_hook,
10624
	.merge_bio_hook = btrfs_merge_bio_hook,
10625
	.readpage_end_io_hook = btrfs_readpage_end_io_hook,
10626
	.writepage_end_io_hook = btrfs_writepage_end_io_hook,
10627
	.writepage_start_hook = btrfs_writepage_start_hook,
10628 10629
	.set_bit_hook = btrfs_set_bit_hook,
	.clear_bit_hook = btrfs_clear_bit_hook,
Josef Bacik's avatar
Josef Bacik committed
10630 10631
	.merge_extent_hook = btrfs_merge_extent_hook,
	.split_extent_hook = btrfs_split_extent_hook,
10632 10633
};

10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645
/*
 * btrfs doesn't support the bmap operation because swapfiles
 * use bmap to make a mapping of extents in the file.  They assume
 * these extents won't change over the life of the file and they
 * use the bmap result to do IO directly to the drive.
 *
 * the btrfs bmap call would return logical addresses that aren't
 * suitable for IO and they also will change frequently as COW
 * operations happen.  So, swapfile + btrfs == corruption.
 *
 * For now we're avoiding this by dropping bmap.
 */
10646
static const struct address_space_operations btrfs_aops = {
Chris Mason's avatar
Chris Mason committed
10647 10648
	.readpage	= btrfs_readpage,
	.writepage	= btrfs_writepage,
Chris Mason's avatar
Chris Mason committed
10649
	.writepages	= btrfs_writepages,
Chris Mason's avatar
Chris Mason committed
10650
	.readpages	= btrfs_readpages,
10651
	.direct_IO	= btrfs_direct_IO,
10652 10653
	.invalidatepage = btrfs_invalidatepage,
	.releasepage	= btrfs_releasepage,
10654
	.set_page_dirty	= btrfs_set_page_dirty,
10655
	.error_remove_page = generic_error_remove_page,
Chris Mason's avatar
Chris Mason committed
10656 10657
};

10658
static const struct address_space_operations btrfs_symlink_aops = {
Chris Mason's avatar
Chris Mason committed
10659 10660
	.readpage	= btrfs_readpage,
	.writepage	= btrfs_writepage,
Chris Mason's avatar
Chris Mason committed
10661 10662
	.invalidatepage = btrfs_invalidatepage,
	.releasepage	= btrfs_releasepage,
Chris Mason's avatar
Chris Mason committed
10663 10664
};

10665
static const struct inode_operations btrfs_file_inode_operations = {
Chris Mason's avatar
Chris Mason committed
10666 10667
	.getattr	= btrfs_getattr,
	.setattr	= btrfs_setattr,
Josef Bacik's avatar
Josef Bacik committed
10668
	.listxattr      = btrfs_listxattr,
Yan's avatar
Yan committed
10669
	.permission	= btrfs_permission,
Yehuda Sadeh's avatar
Yehuda Sadeh committed
10670
	.fiemap		= btrfs_fiemap,
10671
	.get_acl	= btrfs_get_acl,
10672
	.set_acl	= btrfs_set_acl,
10673
	.update_time	= btrfs_update_time,
Chris Mason's avatar
Chris Mason committed
10674
};
10675
static const struct inode_operations btrfs_special_inode_operations = {
Josef Bacik's avatar
Josef Bacik committed
10676 10677
	.getattr	= btrfs_getattr,
	.setattr	= btrfs_setattr,
Yan's avatar
Yan committed
10678
	.permission	= btrfs_permission,
Josef Bacik's avatar
Josef Bacik committed
10679
	.listxattr	= btrfs_listxattr,
10680
	.get_acl	= btrfs_get_acl,
10681
	.set_acl	= btrfs_set_acl,
10682
	.update_time	= btrfs_update_time,
Josef Bacik's avatar
Josef Bacik committed
10683
};
10684
static const struct inode_operations btrfs_symlink_inode_operations = {
Chris Mason's avatar
Chris Mason committed
10685
	.readlink	= generic_readlink,
10686
	.get_link	= page_get_link,
10687
	.getattr	= btrfs_getattr,
10688
	.setattr	= btrfs_setattr,
Yan's avatar
Yan committed
10689
	.permission	= btrfs_permission,
Jim Owens's avatar
Jim Owens committed
10690
	.listxattr	= btrfs_listxattr,
10691
	.update_time	= btrfs_update_time,
Chris Mason's avatar
Chris Mason committed
10692
};
10693

10694
const struct dentry_operations btrfs_dentry_operations = {
10695
	.d_delete	= btrfs_dentry_delete,
10696
	.d_release	= btrfs_dentry_release,
10697
};