data.c 65.6 KB
Newer Older
1
/*
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
 * fs/f2fs/data.c
 *
 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
 *             http://www.samsung.com/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/buffer_head.h>
#include <linux/mpage.h>
#include <linux/writeback.h>
#include <linux/backing-dev.h>
17
#include <linux/pagevec.h>
18 19
#include <linux/blkdev.h>
#include <linux/bio.h>
20
#include <linux/prefetch.h>
21
#include <linux/uio.h>
22
#include <linux/cleancache.h>
23
#include <linux/sched/signal.h>
24 25 26 27

#include "f2fs.h"
#include "node.h"
#include "segment.h"
28
#include "trace.h"
29
#include <trace/events/f2fs.h>
30

31 32 33 34 35
#define NUM_PREALLOC_POST_READ_CTXS	128

static struct kmem_cache *bio_post_read_ctx_cache;
static mempool_t *bio_post_read_ctx_pool;

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
static bool __is_cp_guaranteed(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct inode *inode;
	struct f2fs_sb_info *sbi;

	if (!mapping)
		return false;

	inode = mapping->host;
	sbi = F2FS_I_SB(inode);

	if (inode->i_ino == F2FS_META_INO(sbi) ||
			inode->i_ino ==  F2FS_NODE_INO(sbi) ||
			S_ISDIR(inode->i_mode) ||
51 52
			(S_ISREG(inode->i_mode) &&
			is_inode_flag_set(inode, FI_ATOMIC_FILE)) ||
53 54 55 56 57
			is_cold_data(page))
		return true;
	return false;
}

58 59 60 61 62 63 64 65 66 67 68 69 70 71
/* postprocessing steps for read bios */
enum bio_post_read_step {
	STEP_INITIAL = 0,
	STEP_DECRYPT,
};

struct bio_post_read_ctx {
	struct bio *bio;
	struct work_struct work;
	unsigned int cur_step;
	unsigned int enabled_steps;
};

static void __read_end_io(struct bio *bio)
72
{
73 74
	struct page *page;
	struct bio_vec *bv;
75
	int i;
76

77 78 79 80 81 82
	bio_for_each_segment_all(bv, bio, i) {
		page = bv->bv_page;

		/* PG_error was set if any post_read step failed */
		if (bio->bi_status || PageError(page)) {
			ClearPageUptodate(page);
83 84
			/* will re-read again later */
			ClearPageError(page);
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
		} else {
			SetPageUptodate(page);
		}
		unlock_page(page);
	}
	if (bio->bi_private)
		mempool_free(bio->bi_private, bio_post_read_ctx_pool);
	bio_put(bio);
}

static void bio_post_read_processing(struct bio_post_read_ctx *ctx);

static void decrypt_work(struct work_struct *work)
{
	struct bio_post_read_ctx *ctx =
		container_of(work, struct bio_post_read_ctx, work);

	fscrypt_decrypt_bio(ctx->bio);

	bio_post_read_processing(ctx);
}

static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
{
	switch (++ctx->cur_step) {
	case STEP_DECRYPT:
		if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
			INIT_WORK(&ctx->work, decrypt_work);
			fscrypt_enqueue_decrypt_work(&ctx->work);
			return;
		}
		ctx->cur_step++;
		/* fall-through */
	default:
		__read_end_io(ctx->bio);
	}
}

static bool f2fs_bio_post_read_required(struct bio *bio)
{
	return bio->bi_private && !bio->bi_status;
}

static void f2fs_read_end_io(struct bio *bio)
{
130
	if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), FAULT_IO)) {
131
		f2fs_show_injection_info(FAULT_IO);
132
		bio->bi_status = BLK_STS_IOERR;
133
	}
Chao Yu's avatar
Chao Yu committed
134

135 136
	if (f2fs_bio_post_read_required(bio)) {
		struct bio_post_read_ctx *ctx = bio->bi_private;
137

138 139 140
		ctx->cur_step = STEP_INITIAL;
		bio_post_read_processing(ctx);
		return;
141
	}
142 143

	__read_end_io(bio);
144 145
}

146
static void f2fs_write_end_io(struct bio *bio)
147
{
148
	struct f2fs_sb_info *sbi = bio->bi_private;
149 150
	struct bio_vec *bvec;
	int i;
151

152
	bio_for_each_segment_all(bvec, bio, i) {
153
		struct page *page = bvec->bv_page;
154
		enum count_type type = WB_DATA_TYPE(page);
155

156 157 158 159 160 161
		if (IS_DUMMY_WRITTEN_PAGE(page)) {
			set_page_private(page, (unsigned long)NULL);
			ClearPagePrivate(page);
			unlock_page(page);
			mempool_free(page, sbi->write_io_dummy);

162
			if (unlikely(bio->bi_status))
163 164 165 166
				f2fs_stop_checkpoint(sbi, true);
			continue;
		}

167
		fscrypt_pullback_bio_page(&page, true);
168

169
		if (unlikely(bio->bi_status)) {
170
			mapping_set_error(page->mapping, -EIO);
171 172
			if (type == F2FS_WB_CP_DATA)
				f2fs_stop_checkpoint(sbi, true);
173
		}
174 175 176 177

		f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
					page->index != nid_of_node(page));

178
		dec_page_count(sbi, type);
179 180
		if (f2fs_in_warm_node_list(sbi, page))
			f2fs_del_fsync_node_entry(sbi, page);
181
		clear_cold_data(page);
182
		end_page_writeback(page);
183
	}
184
	if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
185
				wq_has_sleeper(&sbi->cp_wait))
186 187 188 189 190
		wake_up(&sbi->cp_wait);

	bio_put(bio);
}

191 192 193 194 195 196 197 198 199
/*
 * Return true, if pre_bio's bdev is same as its target device.
 */
struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
				block_t blk_addr, struct bio *bio)
{
	struct block_device *bdev = sbi->sb->s_bdev;
	int i;

200 201 202 203 204 205 206 207
	if (f2fs_is_multi_device(sbi)) {
		for (i = 0; i < sbi->s_ndevs; i++) {
			if (FDEV(i).start_blk <= blk_addr &&
			    FDEV(i).end_blk >= blk_addr) {
				blk_addr -= FDEV(i).start_blk;
				bdev = FDEV(i).bdev;
				break;
			}
208 209 210
		}
	}
	if (bio) {
211
		bio_set_dev(bio, bdev);
212 213 214 215 216 217 218 219 220
		bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
	}
	return bdev;
}

int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
{
	int i;

221 222 223
	if (!f2fs_is_multi_device(sbi))
		return 0;

224 225 226 227 228 229 230 231 232
	for (i = 0; i < sbi->s_ndevs; i++)
		if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
			return i;
	return 0;
}

static bool __same_bdev(struct f2fs_sb_info *sbi,
				block_t blk_addr, struct bio *bio)
{
233 234
	struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
	return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
235 236
}

237 238 239 240
/*
 * Low-level block read/write IO operations.
 */
static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
241
				struct writeback_control *wbc,
242 243
				int npages, bool is_read,
				enum page_type type, enum temp_type temp)
244 245 246
{
	struct bio *bio;

247
	bio = f2fs_bio_alloc(sbi, npages, true);
248

249
	f2fs_target_device(sbi, blk_addr, bio);
250 251 252 253 254 255
	if (is_read) {
		bio->bi_end_io = f2fs_read_end_io;
		bio->bi_private = NULL;
	} else {
		bio->bi_end_io = f2fs_write_end_io;
		bio->bi_private = sbi;
Chao Yu's avatar
Chao Yu committed
256
		bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, type, temp);
257
	}
258 259
	if (wbc)
		wbc_init_bio(wbc, bio);
260 261 262 263

	return bio;
}

264 265
static inline void __submit_bio(struct f2fs_sb_info *sbi,
				struct bio *bio, enum page_type type)
266
{
267
	if (!is_read_io(bio_op(bio))) {
268 269 270 271 272
		unsigned int start;

		if (type != DATA && type != NODE)
			goto submit_io;

273
		if (test_opt(sbi, LFS) && current->plug)
274 275
			blk_finish_plug(current->plug);

276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
		start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
		start %= F2FS_IO_SIZE(sbi);

		if (start == 0)
			goto submit_io;

		/* fill dummy pages */
		for (; start < F2FS_IO_SIZE(sbi); start++) {
			struct page *page =
				mempool_alloc(sbi->write_io_dummy,
					GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
			f2fs_bug_on(sbi, !page);

			SetPagePrivate(page);
			set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
			lock_page(page);
			if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
				f2fs_bug_on(sbi, 1);
		}
		/*
		 * In the NODE case, we lose next block address chain. So, we
		 * need to do checkpoint in f2fs_sync_file.
		 */
		if (type == NODE)
			set_sbi_flag(sbi, SBI_NEED_CP);
301
	}
302
submit_io:
303 304 305 306
	if (is_read_io(bio_op(bio)))
		trace_f2fs_submit_read_bio(sbi->sb, type, bio);
	else
		trace_f2fs_submit_write_bio(sbi->sb, type, bio);
307
	submit_bio(bio);
308 309
}

310
static void __submit_merged_bio(struct f2fs_bio_info *io)
311
{
312
	struct f2fs_io_info *fio = &io->fio;
313 314 315 316

	if (!io->bio)
		return;

317 318
	bio_set_op_attrs(io->bio, fio->op, fio->op_flags);

Mike Christie's avatar
Mike Christie committed
319
	if (is_read_io(fio->op))
320
		trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
321
	else
322
		trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
Mike Christie's avatar
Mike Christie committed
323

324
	__submit_bio(io->sbi, io->bio, fio->type);
325 326 327
	io->bio = NULL;
}

328 329
static bool __has_merged_page(struct f2fs_bio_info *io,
				struct inode *inode, nid_t ino, pgoff_t idx)
Chao Yu's avatar
Chao Yu committed
330 331 332 333 334
{
	struct bio_vec *bvec;
	struct page *target;
	int i;

335
	if (!io->bio)
Chao Yu's avatar
Chao Yu committed
336
		return false;
337

338
	if (!inode && !ino)
339
		return true;
Chao Yu's avatar
Chao Yu committed
340 341 342

	bio_for_each_segment_all(bvec, io->bio, i) {

343
		if (bvec->bv_page->mapping)
Chao Yu's avatar
Chao Yu committed
344
			target = bvec->bv_page;
345 346
		else
			target = fscrypt_control_page(bvec->bv_page);
Chao Yu's avatar
Chao Yu committed
347

348 349 350
		if (idx != target->index)
			continue;

351 352 353
		if (inode && inode == target->mapping->host)
			return true;
		if (ino && ino == ino_of_node(target))
Chao Yu's avatar
Chao Yu committed
354 355 356 357 358 359
			return true;
	}

	return false;
}

360
static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
361
				nid_t ino, pgoff_t idx, enum page_type type)
362 363
{
	enum page_type btype = PAGE_TYPE_OF_BIO(type);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
364 365 366
	enum temp_type temp;
	struct f2fs_bio_info *io;
	bool ret = false;
367

Jaegeuk Kim's avatar
Jaegeuk Kim committed
368 369 370 371 372 373
	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
		io = sbi->write_io[btype] + temp;

		down_read(&io->io_rwsem);
		ret = __has_merged_page(io, inode, ino, idx);
		up_read(&io->io_rwsem);
374

Jaegeuk Kim's avatar
Jaegeuk Kim committed
375 376 377 378
		/* TODO: use HOT temp only for meta pages now. */
		if (ret || btype == META)
			break;
	}
379 380 381
	return ret;
}

382
static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
Jaegeuk Kim's avatar
Jaegeuk Kim committed
383
				enum page_type type, enum temp_type temp)
384 385
{
	enum page_type btype = PAGE_TYPE_OF_BIO(type);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
386
	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
387

388
	down_write(&io->io_rwsem);
389 390 391 392

	/* change META to META_FLUSH in the checkpoint procedure */
	if (type >= META_FLUSH) {
		io->fio.type = META_FLUSH;
Mike Christie's avatar
Mike Christie committed
393
		io->fio.op = REQ_OP_WRITE;
394
		io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
395
		if (!test_opt(sbi, NOBARRIER))
396
			io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
397 398
	}
	__submit_merged_bio(io);
399
	up_write(&io->io_rwsem);
400 401
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
402 403 404
static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
				struct inode *inode, nid_t ino, pgoff_t idx,
				enum page_type type, bool force)
405
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
406 407 408 409 410 411 412 413 414 415 416 417 418
	enum temp_type temp;

	if (!force && !has_merged_page(sbi, inode, ino, idx, type))
		return;

	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {

		__f2fs_submit_merged_write(sbi, type, temp);

		/* TODO: use HOT temp only for meta pages now. */
		if (type >= META)
			break;
	}
419 420
}

421
void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
422
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
423
	__submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
424 425
}

426
void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
427
				struct inode *inode, nid_t ino, pgoff_t idx,
428
				enum page_type type)
429
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
430
	__submit_merged_write_cond(sbi, inode, ino, idx, type, false);
431 432
}

433
void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
434
{
435 436 437
	f2fs_submit_merged_write(sbi, DATA);
	f2fs_submit_merged_write(sbi, NODE);
	f2fs_submit_merged_write(sbi, META);
438 439
}

440 441
/*
 * Fill the locked page with data located in the block address.
442
 * A caller needs to unlock the page on failure.
443
 */
444
int f2fs_submit_page_bio(struct f2fs_io_info *fio)
445 446
{
	struct bio *bio;
447 448
	struct page *page = fio->encrypted_page ?
			fio->encrypted_page : fio->page;
449

450 451
	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
			__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
452
		return -EFSCORRUPTED;
453

454
	trace_f2fs_submit_page_bio(page, fio);
455
	f2fs_trace_ios(fio, 0);
456 457

	/* Allocate a new bio */
458
	bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
459
				1, is_read_io(fio->op), fio->type, fio->temp);
460

461
	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
462 463 464
		bio_put(bio);
		return -EFAULT;
	}
465 466 467 468

	if (fio->io_wbc && !is_read_io(fio->op))
		wbc_account_io(fio->io_wbc, page, PAGE_SIZE);

Mike Christie's avatar
Mike Christie committed
469
	bio_set_op_attrs(bio, fio->op, fio->op_flags);
470

471 472
	if (!is_read_io(fio->op))
		inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
473 474

	__submit_bio(fio->sbi, bio, fio->type);
475 476 477
	return 0;
}

478
void f2fs_submit_page_write(struct f2fs_io_info *fio)
479
{
480
	struct f2fs_sb_info *sbi = fio->sbi;
481
	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
Jaegeuk Kim's avatar
Jaegeuk Kim committed
482
	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
483
	struct page *bio_page;
484

485
	f2fs_bug_on(sbi, is_read_io(fio->op));
486

487 488 489 490 491 492
	down_write(&io->io_rwsem);
next:
	if (fio->in_list) {
		spin_lock(&io->io_lock);
		if (list_empty(&io->io_list)) {
			spin_unlock(&io->io_lock);
493
			goto out;
494 495 496 497 498 499
		}
		fio = list_first_entry(&io->io_list,
						struct f2fs_io_info, list);
		list_del(&fio->list);
		spin_unlock(&io->io_lock);
	}
500

501
	if (__is_valid_data_blkaddr(fio->old_blkaddr))
502 503
		verify_block_addr(fio, fio->old_blkaddr);
	verify_block_addr(fio, fio->new_blkaddr);
504

505 506
	bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;

507 508
	/* set submitted = true as a return value */
	fio->submitted = true;
509

510
	inc_page_count(sbi, WB_DATA_TYPE(bio_page));
511

512
	if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
513 514
	    (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
			!__same_bdev(sbi, fio->new_blkaddr, io->bio)))
515
		__submit_merged_bio(io);
516 517
alloc_new:
	if (io->bio == NULL) {
518 519
		if ((fio->type == DATA || fio->type == NODE) &&
				fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
520
			dec_page_count(sbi, WB_DATA_TYPE(bio_page));
521 522
			fio->retry = true;
			goto skip;
523
		}
524
		io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
525 526
						BIO_MAX_PAGES, false,
						fio->type, fio->temp);
527
		io->fio = *fio;
528 529
	}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
530
	if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
531
		__submit_merged_bio(io);
532 533 534
		goto alloc_new;
	}

535 536 537
	if (fio->io_wbc)
		wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);

538
	io->last_block_in_bio = fio->new_blkaddr;
539
	f2fs_trace_ios(fio, 0);
540 541

	trace_f2fs_submit_page_write(fio->page, fio);
542
skip:
543 544
	if (fio->in_list)
		goto next;
545
out:
546
	up_write(&io->io_rwsem);
547 548
}

549
static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
550
					unsigned nr_pages, unsigned op_flag)
551 552 553
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct bio *bio;
554 555
	struct bio_post_read_ctx *ctx;
	unsigned int post_read_steps = 0;
556

557 558 559
	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC))
		return ERR_PTR(-EFAULT);

560
	bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
561
	if (!bio)
562 563 564
		return ERR_PTR(-ENOMEM);
	f2fs_target_device(sbi, blkaddr, bio);
	bio->bi_end_io = f2fs_read_end_io;
565
	bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
566

567 568 569 570 571 572 573 574 575 576 577 578 579
	if (f2fs_encrypted_file(inode))
		post_read_steps |= 1 << STEP_DECRYPT;
	if (post_read_steps) {
		ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
		if (!ctx) {
			bio_put(bio);
			return ERR_PTR(-ENOMEM);
		}
		ctx->bio = bio;
		ctx->enabled_steps = post_read_steps;
		bio->bi_private = ctx;
	}

580 581 582 583 584 585 586
	return bio;
}

/* This can handle encryption stuffs */
static int f2fs_submit_page_read(struct inode *inode, struct page *page,
							block_t blkaddr)
{
587
	struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0);
588 589 590 591

	if (IS_ERR(bio))
		return PTR_ERR(bio);

592 593 594
	/* wait for GCed page writeback via META_MAPPING */
	f2fs_wait_on_block_writeback(inode, blkaddr);

595 596 597 598
	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
		bio_put(bio);
		return -EFAULT;
	}
599
	ClearPageError(page);
600 601 602 603
	__submit_bio(F2FS_I_SB(inode), bio, DATA);
	return 0;
}

604 605 606 607
static void __set_data_blkaddr(struct dnode_of_data *dn)
{
	struct f2fs_node *rn = F2FS_NODE(dn->node_page);
	__le32 *addr_array;
608 609 610 611
	int base = 0;

	if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
		base = get_extra_isize(dn->inode);
612 613 614

	/* Get physical address of data block */
	addr_array = blkaddr_in_node(rn);
615
	addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
616 617
}

618
/*
619 620 621 622 623
 * Lock ordering for the change of data block address:
 * ->data_page
 *  ->node_page
 *    update block addresses in the node page
 */
Chao Yu's avatar
Chao Yu committed
624
void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
625
{
626 627 628
	f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
	__set_data_blkaddr(dn);
	if (set_page_dirty(dn->node_page))
629
		dn->node_changed = true;
630 631
}

632 633 634
void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
{
	dn->data_blkaddr = blkaddr;
Chao Yu's avatar
Chao Yu committed
635
	f2fs_set_data_blkaddr(dn);
636 637 638
	f2fs_update_extent_cache(dn);
}

639
/* dn->ofs_in_node will be returned with up-to-date last block pointer */
Chao Yu's avatar
Chao Yu committed
640
int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
641
{
642
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
643
	int err;
644

645 646 647
	if (!count)
		return 0;

648
	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
649
		return -EPERM;
650 651
	if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
		return err;
652

653 654 655 656 657 658
	trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
						dn->ofs_in_node, count);

	f2fs_wait_on_page_writeback(dn->node_page, NODE, true);

	for (; count > 0; dn->ofs_in_node++) {
659 660
		block_t blkaddr = datablock_addr(dn->inode,
					dn->node_page, dn->ofs_in_node);
661 662 663 664 665 666 667 668 669
		if (blkaddr == NULL_ADDR) {
			dn->data_blkaddr = NEW_ADDR;
			__set_data_blkaddr(dn);
			count--;
		}
	}

	if (set_page_dirty(dn->node_page))
		dn->node_changed = true;
670 671 672
	return 0;
}

673
/* Should keep dn->ofs_in_node unchanged */
Chao Yu's avatar
Chao Yu committed
674
int f2fs_reserve_new_block(struct dnode_of_data *dn)
675 676 677 678
{
	unsigned int ofs_in_node = dn->ofs_in_node;
	int ret;

Chao Yu's avatar
Chao Yu committed
679
	ret = f2fs_reserve_new_blocks(dn, 1);
680 681 682 683
	dn->ofs_in_node = ofs_in_node;
	return ret;
}

684 685 686 687 688
int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
{
	bool need_put = dn->inode_page ? false : true;
	int err;

Chao Yu's avatar
Chao Yu committed
689
	err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
690 691
	if (err)
		return err;
692

693
	if (dn->data_blkaddr == NULL_ADDR)
Chao Yu's avatar
Chao Yu committed
694
		err = f2fs_reserve_new_block(dn);
695
	if (err || need_put)
696 697 698 699
		f2fs_put_dnode(dn);
	return err;
}

700
int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
701
{
702
	struct extent_info ei  = {0,0,0};
703
	struct inode *inode = dn->inode;
704

705 706 707
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn->data_blkaddr = ei.blk + index - ei.fofs;
		return 0;
708
	}
709

710
	return f2fs_reserve_block(dn, index);
711 712
}

Chao Yu's avatar
Chao Yu committed
713
struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
Mike Christie's avatar
Mike Christie committed
714
						int op_flags, bool for_write)
715 716 717 718
{
	struct address_space *mapping = inode->i_mapping;
	struct dnode_of_data dn;
	struct page *page;
719
	struct extent_info ei = {0,0,0};
720
	int err;
721

722
	page = f2fs_grab_cache_page(mapping, index, for_write);
723 724 725
	if (!page)
		return ERR_PTR(-ENOMEM);

Chao Yu's avatar
Chao Yu committed
726 727 728 729 730
	if (f2fs_lookup_extent_cache(inode, index, &ei)) {
		dn.data_blkaddr = ei.blk + index - ei.fofs;
		goto got_it;
	}

731
	set_new_dnode(&dn, inode, NULL, NULL, 0);
Chao Yu's avatar
Chao Yu committed
732
	err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
733 734
	if (err)
		goto put_err;
735 736
	f2fs_put_dnode(&dn);

737
	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
738 739
		err = -ENOENT;
		goto put_err;
740
	}
Chao Yu's avatar
Chao Yu committed
741
got_it:
742 743
	if (PageUptodate(page)) {
		unlock_page(page);
744
		return page;
745
	}
746

747 748 749 750
	/*
	 * A new dentry page is allocated but not able to be written, since its
	 * new inode page couldn't be allocated due to -ENOSPC.
	 * In such the case, its blkaddr can be remained as NEW_ADDR.
Chao Yu's avatar
Chao Yu committed
751 752
	 * see, f2fs_add_link -> f2fs_get_new_data_page ->
	 * f2fs_init_inode_metadata.
753 754
	 */
	if (dn.data_blkaddr == NEW_ADDR) {
755
		zero_user_segment(page, 0, PAGE_SIZE);
756 757
		if (!PageUptodate(page))
			SetPageUptodate(page);
758
		unlock_page(page);
759 760
		return page;
	}
761

762
	err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
763
	if (err)
764
		goto put_err;
765
	return page;
766 767 768 769

put_err:
	f2fs_put_page(page, 1);
	return ERR_PTR(err);
770 771
}

Chao Yu's avatar
Chao Yu committed
772
struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
773 774 775 776 777 778 779 780 781
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;

	page = find_get_page(mapping, index);
	if (page && PageUptodate(page))
		return page;
	f2fs_put_page(page, 0);

Chao Yu's avatar
Chao Yu committed
782
	page = f2fs_get_read_data_page(inode, index, 0, false);
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
	if (IS_ERR(page))
		return page;

	if (PageUptodate(page))
		return page;

	wait_on_page_locked(page);
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 0);
		return ERR_PTR(-EIO);
	}
	return page;
}

/*
 * If it tries to access a hole, return an error.
 * Because, the callers, functions in dir.c and GC, should be able to know
 * whether this page exists or not.
 */
Chao Yu's avatar
Chao Yu committed
802
struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
803
							bool for_write)
804 805 806 807
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
repeat:
Chao Yu's avatar
Chao Yu committed
808
	page = f2fs_get_read_data_page(inode, index, 0, for_write);
809 810
	if (IS_ERR(page))
		return page;
811

812
	/* wait for read completion */
813
	lock_page(page);
814
	if (unlikely(page->mapping != mapping)) {
815 816
		f2fs_put_page(page, 1);
		goto repeat;
817
	}
818 819 820 821
	if (unlikely(!PageUptodate(page))) {
		f2fs_put_page(page, 1);
		return ERR_PTR(-EIO);
	}
822 823 824
	return page;
}

825
/*
826 827
 * Caller ensures that this data page is never allocated.
 * A new zero-filled data page is allocated in the page cache.
828
 *
Chao Yu's avatar
Chao Yu committed
829 830
 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 * f2fs_unlock_op().
831 832
 * Note that, ipage is set only by make_empty_dir, and if any error occur,
 * ipage should be released by this function.
833
 */
Chao Yu's avatar
Chao Yu committed
834
struct page *f2fs_get_new_data_page(struct inode *inode,
835
		struct page *ipage, pgoff_t index, bool new_i_size)
836 837 838 839 840
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
	struct dnode_of_data dn;
	int err;
841

842
	page = f2fs_grab_cache_page(mapping, index, true);
843 844 845 846 847 848
	if (!page) {
		/*
		 * before exiting, we should make sure ipage will be released
		 * if any error occur.
		 */
		f2fs_put_page(ipage, 1);
849
		return ERR_PTR(-ENOMEM);
850
	}
851

852
	set_new_dnode(&dn, inode, ipage, NULL, 0);
853
	err = f2fs_reserve_block(&dn, index);
854 855
	if (err) {
		f2fs_put_page(page, 1);
856
		return ERR_PTR(err);
857
	}
858 859
	if (!ipage)
		f2fs_put_dnode(&dn);
860 861

	if (PageUptodate(page))
862
		goto got_it;
863 864

	if (dn.data_blkaddr == NEW_ADDR) {
865
		zero_user_segment(page, 0, PAGE_SIZE);
866 867
		if (!PageUptodate(page))
			SetPageUptodate(page);
868
	} else {
869
		f2fs_put_page(page, 1);
870

871 872
		/* if ipage exists, blkaddr should be NEW_ADDR */
		f2fs_bug_on(F2FS_I_SB(inode), ipage);
Chao Yu's avatar
Chao Yu committed
873
		page = f2fs_get_lock_data_page(inode, index, true);
874
		if (IS_ERR(page))
875
			return page;
876
	}
877
got_it:
878
	if (new_i_size && i_size_read(inode) <
879
				((loff_t)(index + 1) << PAGE_SHIFT))
880
		f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
881 882 883
	return page;
}

884
static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
885
{
886
	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
887 888
	struct f2fs_summary sum;
	struct node_info ni;
889
	block_t old_blkaddr;
890
	pgoff_t fofs;
891
	blkcnt_t count = 1;
892
	int err;
893

894
	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
895
		return -EPERM;
896

897 898 899 900
	err = f2fs_get_node_info(sbi, dn->nid, &ni);
	if (err)
		return err;

901 902
	dn->data_blkaddr = datablock_addr(dn->inode,
				dn->node_page, dn->ofs_in_node);
903 904 905
	if (dn->data_blkaddr == NEW_ADDR)
		goto alloc;

906 907
	if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
		return err;
908

909
alloc:
910
	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
911 912
	old_blkaddr = dn->data_blkaddr;
	f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
913
					&sum, seg_type, NULL, false);
914 915 916
	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
		invalidate_mapping_pages(META_MAPPING(sbi),
					old_blkaddr, old_blkaddr);
Chao Yu's avatar
Chao Yu committed
917
	f2fs_set_data_blkaddr(dn);
918

919
	/* update i_size */
Chao Yu's avatar
Chao Yu committed
920
	fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
921
							dn->ofs_in_node;
922
	if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
923
		f2fs_i_size_write(dn->inode,
924
				((loff_t)(fofs + 1) << PAGE_SHIFT));
925 926 927
	return 0;
}

928
int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
929
{
930
	struct inode *inode = file_inode(iocb->ki_filp);
931
	struct f2fs_map_blocks map;
932
	int flag;
933
	int err = 0;
934
	bool direct_io = iocb->ki_flags & IOCB_DIRECT;
935

936
	/* convert inline data for Direct I/O*/
937
	if (direct_io) {
938 939 940 941 942
		err = f2fs_convert_inline_inode(inode);
		if (err)
			return err;
	}

943 944 945
	if (is_inode_flag_set(inode, FI_NO_PREALLOC))
		return 0;

946
	map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
947 948 949 950 951 952
	map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
	if (map.m_len > map.m_lblk)
		map.m_len -= map.m_lblk;
	else
		map.m_len = 0;

953
	map.m_next_pgofs = NULL;
954
	map.m_next_extent = NULL;
955
	map.m_seg_type = NO_CHECK_TYPE;
956

957
	if (direct_io) {
Chao Yu's avatar
Chao Yu committed
958
		map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint);
959
		flag = f2fs_force_buffered_io(inode, WRITE) ?
960 961 962
					F2FS_GET_BLOCK_PRE_AIO :
					F2FS_GET_BLOCK_PRE_DIO;
		goto map_blocks;
963
	}
964
	if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
965 966 967
		err = f2fs_convert_inline_inode(inode);
		if (err)
			return err;
968
	}
969
	if (f2fs_has_inline_data(inode))
970
		return err;
971 972 973 974 975 976 977 978 979

	flag = F2FS_GET_BLOCK_PRE_AIO;

map_blocks:
	err = f2fs_map_blocks(inode, &map, 1, flag);
	if (map.m_len > 0 && err == -ENOSPC) {
		if (!direct_io)
			set_inode_flag(inode, FI_NO_PREALLOC);
		err = 0;
980
	}
981
	return err;
982 983
}

984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
{
	if (flag == F2FS_GET_BLOCK_PRE_AIO) {
		if (lock)
			down_read(&sbi->node_change);
		else
			up_read(&sbi->node_change);
	} else {
		if (lock)
			f2fs_lock_op(sbi);
		else
			f2fs_unlock_op(sbi);
	}
}

999
/*
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1000 1001
 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
 * f2fs_map_blocks structure.
Chao Yu's avatar
Chao Yu committed
1002 1003 1004 1005 1006
 * If original data blocks are allocated, then give them to blockdev.
 * Otherwise,
 *     a. preallocate requested block addresses
 *     b. do not use extent cache for better performance
 *     c. give the block addresses to blockdev
1007
 */
Chao Yu's avatar
Chao Yu committed
1008
int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
1009
						int create, int flag)
1010
{
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1011
	unsigned int maxblocks = map->m_len;
1012
	struct dnode_of_data dn;
1013
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1014
	int mode = create ? ALLOC_NODE : LOOKUP_NODE;
1015
	pgoff_t pgofs, end_offset, end;
1016
	int err = 0, ofs = 1;
1017 1018
	unsigned int ofs_in_node, last_ofs_in_node;
	blkcnt_t prealloc;
1019
	struct extent_info ei = {0,0,0};
1020
	block_t blkaddr;
1021
	unsigned int start_pgofs;
1022

1023 1024 1025
	if (!maxblocks)
		return 0;

Jaegeuk Kim's avatar
Jaegeuk Kim committed
1026 1027 1028 1029 1030
	map->m_len = 0;
	map->m_flags = 0;

	/* it only supports block size == page size */
	pgofs =	(pgoff_t)map->m_lblk;
1031
	end = pgofs + maxblocks;
1032

1033
	if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1034 1035 1036
		map->m_pblk = ei.blk + pgofs - ei.fofs;
		map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
		map->m_flags = F2FS_MAP_MAPPED;
1037 1038
		if (map->m_next_extent)
			*map->m_next_extent = pgofs + map->m_len;
1039
		goto out;
1040
	}
1041

Chao Yu's avatar
Chao Yu committed
1042
next_dnode:
1043
	if (create)
1044
		__do_map_lock(sbi, flag, true);
1045 1046 1047

	/* When reading holes, we need its node page */
	set_new_dnode(&dn, inode, NULL, NULL, 0);
Chao Yu's avatar
Chao Yu committed
1048
	err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1049
	if (err) {
1050 1051
		if (flag == F2FS_GET_BLOCK_BMAP)
			map->m_pblk = 0;
1052
		if (err == -ENOENT) {
1053
			err = 0;
1054 1055
			if (map->m_next_pgofs)
				*map->m_next_pgofs =
Chao Yu's avatar
Chao Yu committed
1056
					f2fs_get_next_page_offset(&dn, pgofs);
1057 1058
			if (map->m_next_extent)
				*map->m_next_extent =
Chao Yu's avatar
Chao Yu committed
1059
					f2fs_get_next_page_offset(&dn, pgofs);
1060
		}
1061
		goto unlock_out;
1062
	}
Chao Yu's avatar
Chao Yu committed
1063

1064
	start_pgofs = pgofs;
1065
	prealloc = 0;
1066
	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1067
	end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
Chao Yu's avatar
Chao Yu committed
1068 1069

next_block:
1070
	blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
Chao Yu's avatar
Chao Yu committed
1071

1072 1073
	if (__is_valid_data_blkaddr(blkaddr) &&
		!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
1074
		err = -EFSCORRUPTED;
1075 1076 1077
		goto sync_out;
	}

1078
	if (!is_valid_data_blkaddr(sbi, blkaddr)) {
Chao Yu's avatar
Chao Yu committed
1079
		if (create) {
1080 1081
			if (unlikely(f2fs_cp_error(sbi))) {
				err = -EIO;
Chao Yu's avatar
Chao Yu committed
1082
				goto sync_out;
1083
			}
1084
			if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1085 1086 1087 1088
				if (blkaddr == NULL_ADDR) {
					prealloc++;
					last_ofs_in_node = dn.ofs_in_node;
				}
1089
			} else {
1090 1091
				err = __allocate_data_block(&dn,
							map->m_seg_type);
1092
				if (!err)
1093
					set_inode_flag(inode, FI_APPEND_WRITE);
1094
			}
Chao Yu's avatar
Chao Yu committed
1095
			if (err)
Chao Yu's avatar
Chao Yu committed
1096
				goto sync_out;
1097
			map->m_flags |= F2FS_MAP_NEW;
Chao Yu's avatar
Chao Yu committed
1098
			blkaddr = dn.data_blkaddr;
Chao Yu's avatar
Chao Yu committed
1099
		} else {
1100 1101 1102 1103
			if (flag == F2FS_GET_BLOCK_BMAP) {
				map->m_pblk = 0;
				goto sync_out;
			}
1104 1105
			if (flag == F2FS_GET_BLOCK_PRECACHE)
				goto sync_out;
1106 1107 1108 1109
			if (flag == F2FS_GET_BLOCK_FIEMAP &&
						blkaddr == NULL_ADDR) {
				if (map->m_next_pgofs)
					*map->m_next_pgofs = pgofs + 1;
Chao Yu's avatar
Chao Yu committed
1110
				goto sync_out;
1111
			}
1112 1113 1114 1115
			if (flag != F2FS_GET_BLOCK_FIEMAP) {
				/* for defragment case */
				if (map->m_next_pgofs)
					*map->m_next_pgofs = pgofs + 1;
Chao Yu's avatar
Chao Yu committed
1116
				goto sync_out;
1117
			}
1118 1119
		}
	}
1120

1121 1122 1123
	if (flag == F2FS_GET_BLOCK_PRE_AIO)
		goto skip;

Chao Yu's avatar
Chao Yu committed
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
	if (map->m_len == 0) {
		/* preallocated unwritten block should be mapped for fiemap. */
		if (blkaddr == NEW_ADDR)
			map->m_flags |= F2FS_MAP_UNWRITTEN;
		map->m_flags |= F2FS_MAP_MAPPED;

		map->m_pblk = blkaddr;
		map->m_len = 1;
	} else if ((map->m_pblk != NEW_ADDR &&
			blkaddr == (map->m_pblk + ofs)) ||
1134
			(map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1135
			flag == F2FS_GET_BLOCK_PRE_DIO) {
Chao Yu's avatar
Chao Yu committed
1136 1137 1138 1139 1140
		ofs++;
		map->m_len++;
	} else {
		goto sync_out;
	}
1141

1142
skip:
1143 1144 1145
	dn.ofs_in_node++;
	pgofs++;

1146 1147 1148
	/* preallocate blocks in batch for one dnode page */
	if (flag == F2FS_GET_BLOCK_PRE_AIO &&
			(pgofs == end || dn.ofs_in_node == end_offset)) {
1149

1150
		dn.ofs_in_node = ofs_in_node;
Chao Yu's avatar
Chao Yu committed
1151
		err = f2fs_reserve_new_blocks(&dn, prealloc);
1152 1153
		if (err)
			goto sync_out;
1154

1155 1156 1157 1158
		map->m_len += dn.ofs_in_node - ofs_in_node;
		if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
			err = -ENOSPC;
			goto sync_out;
1159
		}
1160 1161 1162 1163 1164 1165 1166 1167
		dn.ofs_in_node = end_offset;
	}

	if (pgofs >= end)
		goto sync_out;
	else if (dn.ofs_in_node < end_offset)
		goto next_block;

1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	if (flag == F2FS_GET_BLOCK_PRECACHE) {
		if (map->m_flags & F2FS_MAP_MAPPED) {
			unsigned int ofs = start_pgofs - map->m_lblk;

			f2fs_update_extent_cache_range(&dn,
				start_pgofs, map->m_pblk + ofs,
				map->m_len - ofs);
		}
	}

1178 1179 1180
	f2fs_put_dnode(&dn);

	if (create) {
1181
		__do_map_lock(sbi, flag, false);
1182
		f2fs_balance_fs(sbi, dn.node_changed);
1183
	}
1184
	goto next_dnode;
1185

1186
sync_out:
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
	if (flag == F2FS_GET_BLOCK_PRECACHE) {
		if (map->m_flags & F2FS_MAP_MAPPED) {
			unsigned int ofs = start_pgofs - map->m_lblk;

			f2fs_update_extent_cache_range(&dn,
				start_pgofs, map->m_pblk + ofs,
				map->m_len - ofs);
		}
		if (map->m_next_extent)
			*map->m_next_extent = pgofs + 1;
	}
1198
	f2fs_put_dnode(&dn);
1199
unlock_out:
1200
	if (create) {
1201
		__do_map_lock(sbi, flag, false);
1202
		f2fs_balance_fs(sbi, dn.node_changed);
1203
	}
1204
out:
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1205
	trace_f2fs_map_blocks(inode, map, err);
1206
	return err;
1207 1208
}

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
{
	struct f2fs_map_blocks map;
	block_t last_lblk;
	int err;

	if (pos + len > i_size_read(inode))
		return false;

	map.m_lblk = F2FS_BYTES_TO_BLK(pos);
	map.m_next_pgofs = NULL;
	map.m_next_extent = NULL;
	map.m_seg_type = NO_CHECK_TYPE;
	last_lblk = F2FS_BLK_ALIGN(pos + len);

	while (map.m_lblk < last_lblk) {
		map.m_len = last_lblk - map.m_lblk;
		err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
		if (err || map.m_len == 0)
			return false;
		map.m_lblk += map.m_len;
	}
	return true;
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
1234
static int __get_data_block(struct inode *inode, sector_t iblock,
1235
			struct buffer_head *bh, int create, int flag,
1236
			pgoff_t *next_pgofs, int seg_type)
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1237 1238
{
	struct f2fs_map_blocks map;
1239
	int err;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1240 1241 1242

	map.m_lblk = iblock;
	map.m_len = bh->b_size >> inode->i_blkbits;
1243
	map.m_next_pgofs = next_pgofs;
1244
	map.m_next_extent = NULL;
1245
	map.m_seg_type = seg_type;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1246

1247 1248
	err = f2fs_map_blocks(inode, &map, create, flag);
	if (!err) {
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1249 1250
		map_bh(bh, inode->i_sb, map.m_pblk);
		bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1251
		bh->b_size = (u64)map.m_len << inode->i_blkbits;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1252
	}
1253
	return err;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1254 1255
}

1256
static int get_data_block(struct inode *inode, sector_t iblock,
1257 1258
			struct buffer_head *bh_result, int create, int flag,
			pgoff_t *next_pgofs)
1259
{
1260
	return __get_data_block(inode, iblock, bh_result, create,
1261 1262
							flag, next_pgofs,
							NO_CHECK_TYPE);
1263 1264 1265
}

static int get_data_block_dio(struct inode *inode, sector_t iblock,
1266 1267
			struct buffer_head *bh_result, int create)
{
1268
	return __get_data_block(inode, iblock, bh_result, create,
1269
						F2FS_GET_BLOCK_DEFAULT, NULL,
Chao Yu's avatar
Chao Yu committed
1270
						f2fs_rw_hint_to_seg_type(
1271
							inode->i_write_hint));
1272 1273
}

1274
static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1275 1276
			struct buffer_head *bh_result, int create)
{
1277
	/* Block number less than F2FS MAX BLOCKS */
1278
	if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1279 1280
		return -EFBIG;

1281
	return __get_data_block(inode, iblock, bh_result, create,
1282 1283
						F2FS_GET_BLOCK_BMAP, NULL,
						NO_CHECK_TYPE);
1284 1285
}

1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
{
	return (offset >> inode->i_blkbits);
}

static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
{
	return (blk << inode->i_blkbits);
}

Chao Yu's avatar
Chao Yu committed
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
static int f2fs_xattr_fiemap(struct inode *inode,
				struct fiemap_extent_info *fieinfo)
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	struct page *page;
	struct node_info ni;
	__u64 phys = 0, len;
	__u32 flags;
	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
	int err = 0;

	if (f2fs_has_inline_xattr(inode)) {
		int offset;

		page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
						inode->i_ino, false);
		if (!page)
			return -ENOMEM;

1315 1316 1317 1318 1319
		err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
		if (err) {
			f2fs_put_page(page, 1);
			return err;
		}
Chao Yu's avatar
Chao Yu committed
1320 1321 1322 1323

		phys = (__u64)blk_to_logical(inode, ni.blk_addr);
		offset = offsetof(struct f2fs_inode, i_addr) +
					sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1324
					get_inline_xattr_addrs(inode));
Chao Yu's avatar
Chao Yu committed
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345

		phys += offset;
		len = inline_xattr_size(inode);

		f2fs_put_page(page, 1);

		flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;

		if (!xnid)
			flags |= FIEMAP_EXTENT_LAST;

		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
		if (err || err == 1)
			return err;
	}

	if (xnid) {
		page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
		if (!page)
			return -ENOMEM;

1346 1347 1348 1349 1350
		err = f2fs_get_node_info(sbi, xnid, &ni);
		if (err) {
			f2fs_put_page(page, 1);
			return err;
		}
Chao Yu's avatar
Chao Yu committed
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365

		phys = (__u64)blk_to_logical(inode, ni.blk_addr);
		len = inode->i_sb->s_blocksize;

		f2fs_put_page(page, 1);

		flags = FIEMAP_EXTENT_LAST;
	}

	if (phys)
		err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);

	return (err < 0 ? err : 0);
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
1366 1367 1368
int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
		u64 start, u64 len)
{
1369 1370
	struct buffer_head map_bh;
	sector_t start_blk, last_blk;
1371
	pgoff_t next_pgofs;
1372 1373 1374 1375
	u64 logical = 0, phys = 0, size = 0;
	u32 flags = 0;
	int ret = 0;

1376 1377 1378 1379 1380 1381
	if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
		ret = f2fs_precache_extents(inode);
		if (ret)
			return ret;
	}

Chao Yu's avatar
Chao Yu committed
1382
	ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
1383 1384 1385
	if (ret)
		return ret;

1386 1387
	inode_lock(inode);

Chao Yu's avatar
Chao Yu committed
1388 1389 1390 1391 1392
	if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
		ret = f2fs_xattr_fiemap(inode, fieinfo);
		goto out;
	}

1393 1394 1395
	if (f2fs_has_inline_data(inode)) {
		ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
		if (ret != -EAGAIN)
1396
			goto out;
1397 1398
	}

1399 1400 1401 1402 1403
	if (logical_to_blk(inode, len) == 0)
		len = blk_to_logical(inode, 1);

	start_blk = logical_to_blk(inode, start);
	last_blk = logical_to_blk(inode, start + len - 1);
1404

1405 1406 1407 1408
next:
	memset(&map_bh, 0, sizeof(struct buffer_head));
	map_bh.b_size = len;

1409
	ret = get_data_block(inode, start_blk, &map_bh, 0,
1410
					F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1411 1412 1413 1414 1415
	if (ret)
		goto out;

	/* HOLE */
	if (!buffer_mapped(&map_bh)) {
1416
		start_blk = next_pgofs;
1417 1418 1419

		if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
					F2FS_I_SB(inode)->max_file_blocks))
1420
			goto prep_next;
1421

1422 1423
		flags |= FIEMAP_EXTENT_LAST;
	}
1424

1425 1426 1427 1428
	if (size) {
		if (f2fs_encrypted_inode(inode))
			flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;

1429 1430
		ret = fiemap_fill_next_extent(fieinfo, logical,
				phys, size, flags);
1431
	}
1432

1433 1434
	if (start_blk > last_blk || ret)
		goto out;
1435

1436 1437 1438 1439 1440 1441
	logical = blk_to_logical(inode, start_blk);
	phys = blk_to_logical(inode, map_bh.b_blocknr);
	size = map_bh.b_size;
	flags = 0;
	if (buffer_unwritten(&map_bh))
		flags = FIEMAP_EXTENT_UNWRITTEN;
1442

1443
	start_blk += logical_to_blk(inode, size);
1444

1445
prep_next:
1446 1447 1448 1449 1450 1451 1452 1453 1454
	cond_resched();
	if (fatal_signal_pending(current))
		ret = -EINTR;
	else
		goto next;
out:
	if (ret == 1)
		ret = 0;

Al Viro's avatar
Al Viro committed
1455
	inode_unlock(inode);
1456
	return ret;
Jaegeuk Kim's avatar
Jaegeuk Kim committed
1457 1458
}

1459 1460 1461
/*
 * This function was originally taken from fs/mpage.c, and customized for f2fs.
 * Major change was from block_size == page_size in f2fs by default.
1462 1463 1464 1465 1466
 *
 * Note that the aops->readpages() function is ONLY used for read-ahead. If
 * this function ever deviates from doing just read-ahead, it should either
 * use ->readpage() or do the necessary surgery to decouple ->readpages()
 * from read-ahead.
1467 1468 1469
 */
static int f2fs_mpage_readpages(struct address_space *mapping,
			struct list_head *pages, struct page *page,
1470
			unsigned nr_pages, bool is_readahead)
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
{
	struct bio *bio = NULL;
	sector_t last_block_in_bio = 0;
	struct inode *inode = mapping->host;
	const unsigned blkbits = inode->i_blkbits;
	const unsigned blocksize = 1 << blkbits;
	sector_t block_in_file;
	sector_t last_block;
	sector_t last_block_in_file;
	sector_t block_nr;
	struct f2fs_map_blocks map;

	map.m_pblk = 0;
	map.m_lblk = 0;
	map.m_len = 0;
	map.m_flags = 0;
1487
	map.m_next_pgofs = NULL;
1488
	map.m_next_extent = NULL;
1489
	map.m_seg_type = NO_CHECK_TYPE;
1490

LiFan's avatar
LiFan committed
1491
	for (; nr_pages; nr_pages--) {
1492
		if (pages) {
1493
			page = list_last_entry(pages, struct page, lru);
1494 1495

			prefetchw(&page->flags);
1496 1497
			list_del(&page->lru);
			if (add_to_page_cache_lru(page, mapping,
1498 1499
						  page->index,
						  readahead_gfp_mask(mapping)))
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
				goto next_page;
		}

		block_in_file = (sector_t)page->index;
		last_block = block_in_file + nr_pages;
		last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
								blkbits;
		if (last_block > last_block_in_file)
			last_block = last_block_in_file;

		/*
		 * Map blocks using the previous result first.
		 */
		if ((map.m_flags & F2FS_MAP_MAPPED) &&
				block_in_file > map.m_lblk &&
				block_in_file < (map.m_lblk + map.m_len))
			goto got_it;

		/*
		 * Then do more f2fs_map_blocks() calls until we are
		 * done with this page.
		 */
		map.m_flags = 0;

		if (block_in_file < last_block) {
			map.m_lblk = block_in_file;
			map.m_len = last_block - block_in_file;

1528
			if (f2fs_map_blocks(inode, &map, 0,
1529
						F2FS_GET_BLOCK_DEFAULT))
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540
				goto set_error_page;
		}
got_it:
		if ((map.m_flags & F2FS_MAP_MAPPED)) {
			block_nr = map.m_pblk + block_in_file - map.m_lblk;
			SetPageMappedToDisk(page);

			if (!PageUptodate(page) && !cleancache_get_page(page)) {
				SetPageUptodate(page);
				goto confused;
			}
1541 1542 1543 1544

			if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
								DATA_GENERIC))
				goto set_error_page;
1545
		} else {
1546
			zero_user_segment(page, 0, PAGE_SIZE);
1547 1548
			if (!PageUptodate(page))
				SetPageUptodate(page);
1549 1550 1551 1552 1553 1554 1555 1556
			unlock_page(page);
			goto next_page;
		}

		/*
		 * This page will go to BIO.  Do we need to send this
		 * BIO off first?
		 */
1557 1558
		if (bio && (last_block_in_bio != block_nr - 1 ||
			!__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1559
submit_and_realloc:
1560
			__submit_bio(F2FS_I_SB(inode), bio, DATA);
1561 1562 1563
			bio = NULL;
		}
		if (bio == NULL) {
1564 1565
			bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
					is_readahead ? REQ_RAHEAD : 0);
1566 1567
			if (IS_ERR(bio)) {
				bio = NULL;
1568
				goto set_error_page;
1569
			}
1570 1571
		}

1572 1573 1574 1575 1576 1577
		/*
		 * If the page is under writeback, we need to wait for
		 * its completion to see the correct decrypted data.
		 */
		f2fs_wait_on_block_writeback(inode, block_nr);

1578 1579 1580
		if (bio_add_page(bio, page, blocksize, 0) < blocksize)
			goto submit_and_realloc;

1581
		ClearPageError(page);
1582 1583 1584 1585
		last_block_in_bio = block_nr;
		goto next_page;
set_error_page:
		SetPageError(page);
1586
		zero_user_segment(page, 0, PAGE_SIZE);
1587 1588 1589 1590
		unlock_page(page);
		goto next_page;
confused:
		if (bio) {
1591
			__submit_bio(F2FS_I_SB(inode), bio, DATA);
1592 1593 1594 1595 1596
			bio = NULL;
		}
		unlock_page(page);
next_page:
		if (pages)
1597
			put_page(page);
1598 1599 1600
	}
	BUG_ON(pages && !list_empty(pages));
	if (bio)
1601
		__submit_bio(F2FS_I_SB(inode), bio, DATA);
1602 1603 1604
	return 0;
}

1605 1606
static int f2fs_read_data_page(struct file *file, struct page *page)
{
1607
	struct inode *inode = page->mapping->host;
1608
	int ret = -EAGAIN;
1609

1610 1611
	trace_f2fs_readpage(page, DATA);

arter97's avatar
arter97 committed
1612
	/* If the file has inline data, try to read it directly */
1613 1614
	if (f2fs_has_inline_data(inode))
		ret = f2fs_read_inline_data(inode, page);
1615
	if (ret == -EAGAIN)
1616
		ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1, false);
1617
	return ret;
1618 1619 1620 1621 1622 1623
}

static int f2fs_read_data_pages(struct file *file,
			struct address_space *mapping,
			struct list_head *pages, unsigned nr_pages)
{
1624
	struct inode *inode = mapping->host;
1625
	struct page *page = list_last_entry(pages, struct page, lru);
1626 1627

	trace_f2fs_readpages(inode, page, nr_pages);
1628 1629 1630 1631 1632

	/* If the file has inline data, skip readpages */
	if (f2fs_has_inline_data(inode))
		return 0;

1633
	return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true);
1634 1635
}

1636 1637 1638
static int encrypt_one_page(struct f2fs_io_info *fio)
{
	struct inode *inode = fio->page->mapping->host;
1639
	struct page *mpage;
1640 1641
	gfp_t gfp_flags = GFP_NOFS;

1642
	if (!f2fs_encrypted_file(inode))
1643 1644
		return 0;

1645
	/* wait for GCed page writeback via META_MAPPING */
1646
	f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
1647 1648 1649 1650

retry_encrypt:
	fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
			PAGE_SIZE, 0, fio->page->index, gfp_flags);
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
	if (IS_ERR(fio->encrypted_page)) {
		/* flush pending IOs and wait for a while in the ENOMEM case */
		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
			f2fs_flush_merged_writes(fio->sbi);
			congestion_wait(BLK_RW_ASYNC, HZ/50);
			gfp_flags |= __GFP_NOFAIL;
			goto retry_encrypt;
		}
		return PTR_ERR(fio->encrypted_page);
	}
1661

1662 1663 1664 1665 1666 1667
	mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
	if (mpage) {
		if (PageUptodate(mpage))
			memcpy(page_address(mpage),
				page_address(fio->encrypted_page), PAGE_SIZE);
		f2fs_put_page(mpage, 1);
1668
	}
1669
	return 0;
1670 1671
}

Chao Yu's avatar
Chao Yu committed
1672 1673
static inline bool check_inplace_update_policy(struct inode *inode,
				struct f2fs_io_info *fio)
1674
{
Chao Yu's avatar
Chao Yu committed
1675 1676
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	unsigned int policy = SM_I(sbi)->ipu_policy;
1677

Chao Yu's avatar
Chao Yu committed
1678 1679
	if (policy & (0x1 << F2FS_IPU_FORCE))
		return true;
Chao Yu's avatar
Chao Yu committed
1680
	if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
Chao Yu's avatar
Chao Yu committed
1681 1682 1683 1684
		return true;
	if (policy & (0x1 << F2FS_IPU_UTIL) &&
			utilization(sbi) > SM_I(sbi)->min_ipu_util)
		return true;
Chao Yu's avatar
Chao Yu committed
1685
	if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
Chao Yu's avatar
Chao Yu committed
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
			utilization(sbi) > SM_I(sbi)->min_ipu_util)
		return true;

	/*
	 * IPU for rewrite async pages
	 */
	if (policy & (0x1 << F2FS_IPU_ASYNC) &&
			fio && fio->op == REQ_OP_WRITE &&
			!(fio->op_flags & REQ_SYNC) &&
			!f2fs_encrypted_inode(inode))
		return true;

	/* this is only set during fdatasync */
	if (policy & (0x1 << F2FS_IPU_FSYNC) &&
			is_inode_flag_set(inode, FI_NEED_IPU))
		return true;

	return false;
}

Chao Yu's avatar
Chao Yu committed
1706
bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
Chao Yu's avatar
Chao Yu committed
1707
{
1708 1709
	if (f2fs_is_pinned_file(inode))
		return true;
Chao Yu's avatar
Chao Yu committed
1710 1711 1712 1713 1714 1715 1716 1717

	/* if this is cold file, we should overwrite to avoid fragmentation */
	if (file_is_cold(inode))
		return true;

	return check_inplace_update_policy(inode, fio);
}

Chao Yu's avatar
Chao Yu committed
1718
bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
Chao Yu's avatar
Chao Yu committed
1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
{
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);

	if (test_opt(sbi, LFS))
		return true;
	if (S_ISDIR(inode->i_mode))
		return true;
	if (f2fs_is_atomic_file(inode))
		return true;
	if (fio) {
		if (is_cold_data(fio->page))
			return true;
		if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
			return true;
	}
	return false;
}

1737 1738 1739 1740
static inline bool need_inplace_update(struct f2fs_io_info *fio)
{
	struct inode *inode = fio->page->mapping->host;

Chao Yu's avatar
Chao Yu committed
1741
	if (f2fs_should_update_outplace(inode, fio))
1742 1743
		return false;

Chao Yu's avatar
Chao Yu committed
1744
	return f2fs_should_update_inplace(inode, fio);
1745 1746
}

Chao Yu's avatar
Chao Yu committed
1747
int f2fs_do_write_data_page(struct f2fs_io_info *fio)
1748
{
1749
	struct page *page = fio->page;
1750 1751
	struct inode *inode = page->mapping->host;
	struct dnode_of_data dn;
1752
	struct extent_info ei = {0,0,0};
1753
	struct node_info ni;
1754
	bool ipu_force = false;
1755 1756 1757
	int err = 0;

	set_new_dnode(&dn, inode, NULL, NULL, 0);
1758 1759 1760
	if (need_inplace_update(fio) &&
			f2fs_lookup_extent_cache(inode, page->index, &ei)) {
		fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1761

1762 1763
		if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
							DATA_GENERIC))
1764
			return -EFSCORRUPTED;
1765 1766 1767 1768

		ipu_force = true;
		fio->need_lock = LOCK_DONE;
		goto got_it;
1769
	}
1770

1771 1772 1773
	/* Deadlock due to between page->lock and f2fs_lock_op */
	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
		return -EAGAIN;
1774

Chao Yu's avatar
Chao Yu committed
1775
	err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1776
	if (err)
1777
		goto out;
1778

1779
	fio->old_blkaddr = dn.data_blkaddr;
1780 1781

	/* This page is already truncated */
1782
	if (fio->old_blkaddr == NULL_ADDR) {
1783
		ClearPageUptodate(page);
1784
		goto out_writepage;
1785
	}
1786
got_it:
1787 1788 1789
	if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
		!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
							DATA_GENERIC)) {
1790
		err = -EFSCORRUPTED;
1791 1792
		goto out_writepage;
	}
1793 1794 1795 1796
	/*
	 * If current allocation needs SSR,
	 * it had better in-place writes for updated data.
	 */
1797
	if (ipu_force || (is_valid_data_blkaddr(fio->sbi, fio->old_blkaddr) &&
1798
					need_inplace_update(fio))) {
1799 1800 1801 1802 1803
		err = encrypt_one_page(fio);
		if (err)
			goto out_writepage;

		set_page_writeback(page);
1804
		ClearPageError(page);
1805
		f2fs_put_dnode(&dn);
1806
		if (fio->need_lock == LOCK_REQ)
1807
			f2fs_unlock_op(fio->sbi);
Chao Yu's avatar
Chao Yu committed
1808
		err = f2fs_inplace_write_data(fio);
1809
		trace_f2fs_do_write_data_page(fio->page, IPU);
1810
		set_inode_flag(inode, FI_UPDATE_WRITE);
1811
		return err;
1812
	}
1813

1814 1815 1816 1817 1818 1819 1820 1821
	if (fio->need_lock == LOCK_RETRY) {
		if (!f2fs_trylock_op(fio->sbi)) {
			err = -EAGAIN;
			goto out_writepage;
		}
		fio->need_lock = LOCK_REQ;
	}

1822 1823 1824 1825 1826 1827
	err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
	if (err)
		goto out_writepage;

	fio->version = ni.version;

1828 1829 1830 1831 1832
	err = encrypt_one_page(fio);
	if (err)
		goto out_writepage;

	set_page_writeback(page);
1833
	ClearPageError(page);
1834

1835
	/* LFS mode write path */
Chao Yu's avatar
Chao Yu committed
1836
	f2fs_outplace_write_data(&dn, fio);
1837 1838 1839 1840
	trace_f2fs_do_write_data_page(page, OPU);
	set_inode_flag(inode, FI_APPEND_WRITE);
	if (page->index == 0)
		set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1841 1842
out_writepage:
	f2fs_put_dnode(&dn);
1843
out:
1844
	if (fio->need_lock == LOCK_REQ)
1845
		f2fs_unlock_op(fio->sbi);
1846 1847 1848
	return err;
}

1849
static int __write_data_page(struct page *page, bool *submitted,
Chao Yu's avatar
Chao Yu committed
1850 1851
				struct writeback_control *wbc,
				enum iostat_type io_type)
1852 1853
{
	struct inode *inode = page->mapping->host;
1854
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1855 1856
	loff_t i_size = i_size_read(inode);
	const pgoff_t end_index = ((unsigned long long) i_size)
1857
							>> PAGE_SHIFT;
1858
	loff_t psize = (page->index + 1) << PAGE_SHIFT;
1859
	unsigned offset = 0;
1860
	bool need_balance_fs = false;
1861
	int err = 0;
1862
	struct f2fs_io_info fio = {
1863
		.sbi = sbi,
1864
		.ino = inode->i_ino,
1865
		.type = DATA,
Mike Christie's avatar
Mike Christie committed
1866
		.op = REQ_OP_WRITE,
1867
		.op_flags = wbc_to_write_flags(wbc),
1868
		.old_blkaddr = NULL_ADDR,
1869
		.page = page,
1870
		.encrypted_page = NULL,
1871
		.submitted = false,
1872
		.need_lock = LOCK_RETRY,
Chao Yu's avatar
Chao Yu committed
1873
		.io_type = io_type,
1874
		.io_wbc = wbc,
1875
	};
1876

1877 1878
	trace_f2fs_writepage(page, DATA);

1879 1880 1881
	/* we should bypass data pages to proceed the kworkder jobs */
	if (unlikely(f2fs_cp_error(sbi))) {
		mapping_set_error(page->mapping, -EIO);
1882 1883 1884 1885 1886 1887
		/*
		 * don't drop any dirty dentry pages for keeping lastest
		 * directory structure.
		 */
		if (S_ISDIR(inode->i_mode))
			goto redirty_out;
1888 1889 1890
		goto out;
	}

1891 1892 1893
	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
		goto redirty_out;

1894
	if (page->index < end_index)
1895
		goto write;
1896 1897 1898 1899 1900

	/*
	 * If the offset is out-of-range of file size,
	 * this page does not have to be written to disk.
	 */
1901
	offset = i_size & (PAGE_SIZE - 1);
1902
	if ((page->index >= end_index + 1) || !offset)
1903
		goto out;
1904

1905
	zero_user_segment(page, offset, PAGE_SIZE);
1906
write:
1907 1908
	if (f2fs_is_drop_cache(inode))
		goto out;
1909 1910 1911
	/* we should not write 0'th page having journal header */
	if (f2fs_is_volatile_file(inode) && (!page->index ||
			(!wbc->for_reclaim &&
Chao Yu's avatar
Chao Yu committed
1912
			f2fs_available_free_memory(sbi, BASE_CHECK))))
1913
		goto redirty_out;
1914

1915
	/* Dentry blocks are controlled by checkpoint */
1916
	if (S_ISDIR(inode->i_mode)) {
1917
		fio.need_lock = LOCK_DONE;
Chao Yu's avatar
Chao Yu committed
1918
		err = f2fs_do_write_data_page(&fio);
1919 1920
		goto done;
	}
1921

1922
	if (!wbc->for_reclaim)
1923
		need_balance_fs = true;
1924
	else if (has_not_enough_free_secs(sbi, 0, 0))
1925
		goto redirty_out;
1926 1927
	else
		set_inode_flag(inode, FI_HOT_DATA);
1928

1929
	err = -EAGAIN;
1930
	if (f2fs_has_inline_data(inode)) {
1931
		err = f2fs_write_inline_data(inode, page);
1932 1933 1934
		if (!err)
			goto out;
	}
1935

1936
	if (err == -EAGAIN) {
Chao Yu's avatar
Chao Yu committed
1937
		err = f2fs_do_write_data_page(&fio);
1938 1939
		if (err == -EAGAIN) {
			fio.need_lock = LOCK_REQ;
Chao Yu's avatar
Chao Yu committed
1940
			err = f2fs_do_write_data_page(&fio);
1941 1942
		}
	}
1943

1944 1945 1946 1947 1948 1949 1950 1951
	if (err) {
		file_set_keep_isize(inode);
	} else {
		down_write(&F2FS_I(inode)->i_sem);
		if (F2FS_I(inode)->last_disk_size < psize)
			F2FS_I(inode)->last_disk_size = psize;
		up_write(&F2FS_I(inode)->i_sem);
	}
1952

1953 1954 1955
done:
	if (err && err != -ENOENT)
		goto redirty_out;
1956

1957
out:
1958
	inode_dec_dirty_pages(inode);
1959 1960
	if (err)
		ClearPageUptodate(page);
1961 1962

	if (wbc->for_reclaim) {
1963
		f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
1964
		clear_inode_flag(inode, FI_HOT_DATA);
Chao Yu's avatar
Chao Yu committed
1965
		f2fs_remove_dirty_inode(inode);
1966
		submitted = NULL;
1967 1968
	}

1969
	unlock_page(page);
1970 1971
	if (!S_ISDIR(inode->i_mode))
		f2fs_balance_fs(sbi, need_balance_fs);
1972

1973
	if (unlikely(f2fs_cp_error(sbi))) {
1974
		f2fs_submit_merged_write(sbi, DATA);
1975 1976 1977 1978 1979
		submitted = NULL;
	}

	if (submitted)
		*submitted = fio.submitted;
1980

1981 1982 1983
	return 0;

redirty_out:
1984
	redirty_page_for_writepage(wbc, page);
1985 1986 1987 1988 1989 1990 1991
	/*
	 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
	 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
	 * file_write_and_wait_range() will see EIO error, which is critical
	 * to return value of fsync() followed by atomic_write failure to user.
	 */
	if (!err || wbc->for_reclaim)
1992
		return AOP_WRITEPAGE_ACTIVATE;
1993 1994
	unlock_page(page);
	return err;
1995 1996
}

1997 1998 1999
static int f2fs_write_data_page(struct page *page,
					struct writeback_control *wbc)
{
Chao Yu's avatar
Chao Yu committed
2000
	return __write_data_page(page, NULL, wbc, FS_DATA_IO);
2001 2002
}

2003 2004 2005 2006 2007 2008
/*
 * This function was copied from write_cche_pages from mm/page-writeback.c.
 * The major change is making write step of cold data page separately from
 * warm/hot data page.
 */
static int f2fs_write_cache_pages(struct address_space *mapping,
Chao Yu's avatar
Chao Yu committed
2009 2010
					struct writeback_control *wbc,
					enum iostat_type io_type)
2011 2012 2013 2014
{
	int ret = 0;
	int done = 0;
	struct pagevec pvec;
2015
	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
2016 2017 2018 2019 2020
	int nr_pages;
	pgoff_t uninitialized_var(writeback_index);
	pgoff_t index;
	pgoff_t end;		/* Inclusive */
	pgoff_t done_index;
2021
	pgoff_t last_idx = ULONG_MAX;
2022 2023 2024 2025
	int cycled;
	int range_whole = 0;
	int tag;

2026
	pagevec_init(&pvec);
2027

2028 2029 2030 2031 2032 2033
	if (get_dirty_pages(mapping->host) <=
				SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
		set_inode_flag(mapping->host, FI_HOT_DATA);
	else
		clear_inode_flag(mapping->host, FI_HOT_DATA);

2034 2035 2036 2037 2038 2039 2040 2041 2042
	if (wbc->range_cyclic) {
		writeback_index = mapping->writeback_index; /* prev offset */
		index = writeback_index;
		if (index == 0)
			cycled = 1;
		else
			cycled = 0;
		end = -1;
	} else {
2043 2044
		index = wbc->range_start >> PAGE_SHIFT;
		end = wbc->range_end >> PAGE_SHIFT;
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
			range_whole = 1;
		cycled = 1; /* ignore range_cyclic tests */
	}
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
		tag = PAGECACHE_TAG_TOWRITE;
	else
		tag = PAGECACHE_TAG_DIRTY;
retry:
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
		tag_pages_for_writeback(mapping, index, end);
	done_index = index;
	while (!done && (index <= end)) {
		int i;

2060
		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2061
				tag);
2062 2063 2064 2065 2066
		if (nr_pages == 0)
			break;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
2067
			bool submitted = false;
2068

2069
			/* give a priority to WB_SYNC threads */
2070
			if (atomic_read(&sbi->wb_sync_req[DATA]) &&
2071 2072 2073 2074 2075
					wbc->sync_mode == WB_SYNC_NONE) {
				done = 1;
				break;
			}

2076
			done_index = page->index;
2077
retry_write:
2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
			lock_page(page);

			if (unlikely(page->mapping != mapping)) {
continue_unlock:
				unlock_page(page);
				continue;
			}

			if (!PageDirty(page)) {
				/* someone wrote it for us */
				goto continue_unlock;
			}

			if (PageWriteback(page)) {
				if (wbc->sync_mode != WB_SYNC_NONE)
2093 2094
					f2fs_wait_on_page_writeback(page,
								DATA, true);
2095 2096 2097 2098 2099 2100 2101 2102
				else
					goto continue_unlock;
			}

			BUG_ON(PageWriteback(page));
			if (!clear_page_dirty_for_io(page))
				goto continue_unlock;

Chao Yu's avatar
Chao Yu committed
2103
			ret = __write_data_page(page, &submitted, wbc, io_type);
2104
			if (unlikely(ret)) {
2105 2106 2107 2108 2109 2110 2111 2112
				/*
				 * keep nr_to_write, since vfs uses this to
				 * get # of written pages.
				 */
				if (ret == AOP_WRITEPAGE_ACTIVATE) {
					unlock_page(page);
					ret = 0;
					continue;
2113 2114 2115 2116 2117 2118 2119 2120 2121
				} else if (ret == -EAGAIN) {
					ret = 0;
					if (wbc->sync_mode == WB_SYNC_ALL) {
						cond_resched();
						congestion_wait(BLK_RW_ASYNC,
									HZ/50);
						goto retry_write;
					}
					continue;
2122
				}
2123 2124 2125
				done_index = page->index + 1;
				done = 1;
				break;
2126
			} else if (submitted) {
2127
				last_idx = page->index;
2128 2129
			}

2130
			if (--wbc->nr_to_write <= 0 &&
2131
					wbc->sync_mode == WB_SYNC_NONE) {
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
				done = 1;
				break;
			}
		}
		pagevec_release(&pvec);
		cond_resched();
	}

	if (!cycled && !done) {
		cycled = 1;
		index = 0;
		end = writeback_index - 1;
		goto retry;
	}
	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
		mapping->writeback_index = done_index;

2149
	if (last_idx != ULONG_MAX)
2150 2151
		f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
						0, last_idx, DATA);
2152

2153 2154 2155
	return ret;
}

2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
static inline bool __should_serialize_io(struct inode *inode,
					struct writeback_control *wbc)
{
	if (!S_ISREG(inode->i_mode))
		return false;
	if (wbc->sync_mode != WB_SYNC_ALL)
		return true;
	if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
		return true;
	return false;
}

2168
static int __f2fs_write_data_pages(struct address_space *mapping,
Chao Yu's avatar
Chao Yu committed
2169 2170
						struct writeback_control *wbc,
						enum iostat_type io_type)
2171 2172
{
	struct inode *inode = mapping->host;
2173
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2174
	struct blk_plug plug;
2175
	int ret;
2176
	bool locked = false;
2177

P J P's avatar
P J P committed
2178 2179 2180 2181
	/* deal with chardevs and other special file */
	if (!mapping->a_ops->writepage)
		return 0;

2182 2183 2184 2185
	/* skip writing if there is no dirty page in this inode */
	if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
		return 0;

2186 2187 2188 2189
	/* during POR, we don't need to trigger writepage at all. */
	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
		goto skip_write;

2190 2191
	if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
			get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
Chao Yu's avatar
Chao Yu committed
2192
			f2fs_available_free_memory(sbi, DIRTY_DENTS))
2193 2194
		goto skip_write;

Chao Yu's avatar
Chao Yu committed
2195
	/* skip writing during file defragment */
2196
	if (is_inode_flag_set(inode, FI_DO_DEFRAG))
Chao Yu's avatar
Chao Yu committed
2197 2198
		goto skip_write;

2199 2200
	trace_f2fs_writepages(mapping->host, wbc, DATA);

2201 2202
	/* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
	if (wbc->sync_mode == WB_SYNC_ALL)
2203 2204
		atomic_inc(&sbi->wb_sync_req[DATA]);
	else if (atomic_read(&sbi->wb_sync_req[DATA]))
2205 2206
		goto skip_write;

2207 2208 2209 2210 2211
	if (__should_serialize_io(inode, wbc)) {
		mutex_lock(&sbi->writepages);
		locked = true;
	}

2212
	blk_start_plug(&plug);
Chao Yu's avatar
Chao Yu committed
2213
	ret = f2fs_write_cache_pages(mapping, wbc, io_type);
2214
	blk_finish_plug(&plug);
2215

2216 2217 2218
	if (locked)
		mutex_unlock(&sbi->writepages);

2219
	if (wbc->sync_mode == WB_SYNC_ALL)
2220
		atomic_dec(&sbi->wb_sync_req[DATA]);
2221 2222 2223 2224
	/*
	 * if some pages were truncated, we cannot guarantee its mapping->host
	 * to detect pending bios.
	 */
2225

Chao Yu's avatar
Chao Yu committed
2226
	f2fs_remove_dirty_inode(inode);
2227
	return ret;
2228 2229

skip_write:
2230
	wbc->pages_skipped += get_dirty_pages(inode);
2231
	trace_f2fs_writepages(mapping->host, wbc, DATA);
2232
	return 0;
2233 2234
}

Chao Yu's avatar
Chao Yu committed
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
static int f2fs_write_data_pages(struct address_space *mapping,
			    struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;

	return __f2fs_write_data_pages(mapping, wbc,
			F2FS_I(inode)->cp_task == current ?
			FS_CP_DATA_IO : FS_DATA_IO);
}

2245 2246 2247
static void f2fs_write_failed(struct address_space *mapping, loff_t to)
{
	struct inode *inode = mapping->host;
2248
	loff_t i_size = i_size_read(inode);
2249

2250
	if (to > i_size) {
2251
		down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
2252
		down_write(&F2FS_I(inode)->i_mmap_sem);
2253

2254
		truncate_pagecache(inode, i_size);
Chao Yu's avatar
Chao Yu committed
2255
		f2fs_truncate_blocks(inode, i_size, true);
2256

2257
		up_write(&F2FS_I(inode)->i_mmap_sem);
2258
		up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
2259 2260 2261
	}
}

2262 2263 2264 2265 2266 2267 2268 2269
static int prepare_write_begin(struct f2fs_sb_info *sbi,
			struct page *page, loff_t pos, unsigned len,
			block_t *blk_addr, bool *node_changed)
{
	struct inode *inode = page->mapping->host;
	pgoff_t index = page->index;
	struct dnode_of_data dn;
	struct page *ipage;
2270
	bool locked = false;
2271
	struct extent_info ei = {0,0,0};
2272
	int err = 0;
2273
	int flag;
2274

2275 2276 2277 2278
	/*
	 * we already allocated all the blocks, so we don't need to get
	 * the block addresses when there is no need to fill the page.
	 */
2279 2280
	if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
			!is_inode_flag_set(inode, FI_NO_PREALLOC))
2281 2282
		return 0;

2283 2284 2285 2286 2287 2288
	/* f2fs_lock_op avoids race between write CP and convert_inline_page */
	if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
		flag = F2FS_GET_BLOCK_DEFAULT;
	else
		flag = F2FS_GET_BLOCK_PRE_AIO;

2289
	if (f2fs_has_inline_data(inode) ||
2290
			(pos & PAGE_MASK) >= i_size_read(inode)) {
2291
		__do_map_lock(sbi, flag, true);
2292 2293 2294
		locked = true;
	}
restart:
2295
	/* check inline_data */
Chao Yu's avatar
Chao Yu committed
2296
	ipage = f2fs_get_node_page(sbi, inode->i_ino);
2297 2298 2299 2300 2301 2302 2303 2304
	if (IS_ERR(ipage)) {
		err = PTR_ERR(ipage);
		goto unlock_out;
	}

	set_new_dnode(&dn, inode, ipage, ipage, 0);

	if (f2fs_has_inline_data(inode)) {
2305
		if (pos + len <= MAX_INLINE_DATA(inode)) {
Chao Yu's avatar
Chao Yu committed
2306
			f2fs_do_read_inline_data(page, ipage);
2307
			set_inode_flag(inode, FI_DATA_EXIST);
2308 2309
			if (inode->i_nlink)
				set_inline_node(ipage);
2310 2311 2312
		} else {
			err = f2fs_convert_inline_page(&dn, page);
			if (err)
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
				goto out;
			if (dn.data_blkaddr == NULL_ADDR)
				err = f2fs_get_block(&dn, index);
		}
	} else if (locked) {
		err = f2fs_get_block(&dn, index);
	} else {
		if (f2fs_lookup_extent_cache(inode, index, &ei)) {
			dn.data_blkaddr = ei.blk + index - ei.fofs;
		} else {
			/* hole case */
Chao Yu's avatar
Chao Yu committed
2324
			err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
2325
			if (err || dn.data_blkaddr == NULL_ADDR) {
2326
				f2fs_put_dnode(&dn);
2327 2328
				__do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
								true);
2329
				WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
2330 2331 2332
				locked = true;
				goto restart;
			}
2333 2334
		}
	}
2335

2336 2337 2338
	/* convert_inline_page can make node_changed */
	*blk_addr = dn.data_blkaddr;
	*node_changed = dn.node_changed;
2339
out:
2340 2341
	f2fs_put_dnode(&dn);
unlock_out:
2342
	if (locked)
2343
		__do_map_lock(sbi, flag, false);
2344 2345 2346
	return err;
}

2347 2348 2349 2350 2351
static int f2fs_write_begin(struct file *file, struct address_space *mapping,
		loff_t pos, unsigned len, unsigned flags,
		struct page **pagep, void **fsdata)
{
	struct inode *inode = mapping->host;
2352
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2353
	struct page *page = NULL;
2354
	pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
2355
	bool need_balance = false, drop_atomic = false;
2356
	block_t blkaddr = NULL_ADDR;
2357 2358
	int err = 0;

2359 2360
	trace_f2fs_write_begin(inode, pos, len, flags);

2361 2362 2363
	if ((f2fs_is_atomic_file(inode) &&
			!f2fs_available_free_memory(sbi, INMEM_PAGES)) ||
			is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) {
2364
		err = -ENOMEM;
2365
		drop_atomic = true;
2366 2367 2368
		goto fail;
	}

2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
	/*
	 * We should check this at this moment to avoid deadlock on inode page
	 * and #0 page. The locking rule for inline_data conversion should be:
	 * lock_page(page #0) -> lock_page(inode_page)
	 */
	if (index != 0) {
		err = f2fs_convert_inline_inode(inode);
		if (err)
			goto fail;
	}
2379
repeat:
2380 2381 2382 2383
	/*
	 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
	 * wait_for_stable_page. Will wait that below with our IO control.
	 */
2384
	page = f2fs_pagecache_get_page(mapping, index,
2385
				FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
2386 2387 2388 2389
	if (!page) {
		err = -ENOMEM;
		goto fail;
	}
2390

2391 2392
	*pagep = page;

2393 2394
	err = prepare_write_begin(sbi, page, pos, len,
					&blkaddr, &need_balance);
2395
	if (err)
2396
		goto fail;
2397

2398
	if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2399
		unlock_page(page);
2400
		f2fs_balance_fs(sbi, true);
2401 2402 2403 2404 2405 2406 2407 2408
		lock_page(page);
		if (page->mapping != mapping) {
			/* The page got truncated from under us */
			f2fs_put_page(page, 1);
			goto repeat;
		}
	}

2409
	f2fs_wait_on_page_writeback(page, DATA, false);
2410

2411 2412
	if (len == PAGE_SIZE || PageUptodate(page))
		return 0;
2413

2414 2415 2416 2417 2418
	if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
		zero_user_segment(page, len, PAGE_SIZE);
		return 0;
	}

2419
	if (blkaddr == NEW_ADDR) {
2420
		zero_user_segment(page, 0, PAGE_SIZE);
2421
		SetPageUptodate(page);
2422
	} else {
2423 2424
		err = f2fs_submit_page_read(inode, page, blkaddr);
		if (err)
2425
			goto fail;
2426

2427
		lock_page(page);
2428
		if (unlikely(page->mapping != mapping)) {
2429 2430
			f2fs_put_page(page, 1);
			goto repeat;
2431
		}
2432 2433 2434
		if (unlikely(!PageUptodate(page))) {
			err = -EIO;
			goto fail;
2435
		}
2436 2437
	}
	return 0;
2438

2439
fail:
2440
	f2fs_put_page(page, 1);
2441
	f2fs_write_failed(mapping, pos + len);
2442
	if (drop_atomic)
Chao Yu's avatar
Chao Yu committed
2443
		f2fs_drop_inmem_pages_all(sbi, false);
2444
	return err;
2445 2446
}

2447 2448 2449 2450 2451 2452 2453
static int f2fs_write_end(struct file *file,
			struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
{
	struct inode *inode = page->mapping->host;

2454 2455
	trace_f2fs_write_end(inode, pos, len, copied);

2456 2457 2458 2459 2460 2461
	/*
	 * This should be come from len == PAGE_SIZE, and we expect copied
	 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
	 * let generic_perform_write() try to copy data again through copied=0.
	 */
	if (!PageUptodate(page)) {
2462
		if (unlikely(copied != len))
2463 2464 2465 2466 2467 2468 2469
			copied = 0;
		else
			SetPageUptodate(page);
	}
	if (!copied)
		goto unlock_out;

2470
	set_page_dirty(page);
2471

2472 2473
	if (pos + copied > i_size_read(inode))
		f2fs_i_size_write(inode, pos + copied);
2474
unlock_out:
2475
	f2fs_put_page(page, 1);
2476
	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2477 2478 2479
	return copied;
}

2480 2481
static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
			   loff_t offset)
2482
{
2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496
	unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
	unsigned blkbits = i_blkbits;
	unsigned blocksize_mask = (1 << blkbits) - 1;
	unsigned long align = offset | iov_iter_alignment(iter);
	struct block_device *bdev = inode->i_sb->s_bdev;

	if (align & blocksize_mask) {
		if (bdev)
			blkbits = blksize_bits(bdev_logical_block_size(bdev));
		blocksize_mask = (1 << blkbits) - 1;
		if (align & blocksize_mask)
			return -EINVAL;
		return 1;
	}
2497 2498 2499
	return 0;
}

2500
static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2501
{
2502
	struct address_space *mapping = iocb->ki_filp->f_mapping;
2503
	struct inode *inode = mapping->host;
2504
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2505
	size_t count = iov_iter_count(iter);
2506
	loff_t offset = iocb->ki_pos;
2507
	int rw = iov_iter_rw(iter);
2508
	int err;
2509
	enum rw_hint hint = iocb->ki_hint;
2510
	int whint_mode = F2FS_OPTION(sbi).whint_mode;
2511

2512
	err = check_direct_IO(inode, iter, offset);
2513
	if (err)
2514
		return err < 0 ? err : 0;
2515

2516
	if (f2fs_force_buffered_io(inode, rw))
2517
		return 0;
2518

2519
	trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2520

2521 2522 2523
	if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
		iocb->ki_hint = WRITE_LIFE_NOT_SET;

2524
	if (!down_read_trylock(&F2FS_I(inode)->i_gc_rwsem[rw])) {
2525 2526 2527 2528 2529
		if (iocb->ki_flags & IOCB_NOWAIT) {
			iocb->ki_hint = hint;
			err = -EAGAIN;
			goto out;
		}
2530
		down_read(&F2FS_I(inode)->i_gc_rwsem[rw]);
2531 2532
	}

2533
	err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
2534
	up_read(&F2FS_I(inode)->i_gc_rwsem[rw]);
2535 2536

	if (rw == WRITE) {
2537 2538
		if (whint_mode == WHINT_MODE_OFF)
			iocb->ki_hint = hint;
Chao Yu's avatar
Chao Yu committed
2539 2540 2541
		if (err > 0) {
			f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
									err);
2542
			set_inode_flag(inode, FI_UPDATE_WRITE);
Chao Yu's avatar
Chao Yu committed
2543
		} else if (err < 0) {
2544
			f2fs_write_failed(mapping, offset + count);
Chao Yu's avatar
Chao Yu committed
2545
		}
2546
	}
2547

2548
out:
2549
	trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2550

2551
	return err;
2552 2553
}

2554 2555
void f2fs_invalidate_page(struct page *page, unsigned int offset,
							unsigned int length)
2556 2557
{
	struct inode *inode = page->mapping->host;
2558
	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2559

2560
	if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2561
		(offset % PAGE_SIZE || length != PAGE_SIZE))
2562 2563
		return;

2564
	if (PageDirty(page)) {
2565
		if (inode->i_ino == F2FS_META_INO(sbi)) {
2566
			dec_page_count(sbi, F2FS_DIRTY_META);
2567
		} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2568
			dec_page_count(sbi, F2FS_DIRTY_NODES);
2569
		} else {
2570
			inode_dec_dirty_pages(inode);
Chao Yu's avatar
Chao Yu committed
2571
			f2fs_remove_dirty_inode(inode);
2572
		}
2573
	}
Chao Yu's avatar
Chao Yu committed
2574 2575 2576

	/* This is atomic written page, keep Private */
	if (IS_ATOMIC_WRITTEN_PAGE(page))
Chao Yu's avatar
Chao Yu committed
2577
		return f2fs_drop_inmem_page(inode, page);
Chao Yu's avatar
Chao Yu committed
2578

2579
	set_page_private(page, 0);
2580 2581 2582
	ClearPagePrivate(page);
}

2583
int f2fs_release_page(struct page *page, gfp_t wait)
2584
{
2585 2586 2587 2588
	/* If this is dirty page, keep PagePrivate */
	if (PageDirty(page))
		return 0;

Chao Yu's avatar
Chao Yu committed
2589 2590 2591 2592
	/* This is atomic written page, keep Private */
	if (IS_ATOMIC_WRITTEN_PAGE(page))
		return 0;

2593
	set_page_private(page, 0);
2594
	ClearPagePrivate(page);
2595
	return 1;
2596 2597 2598 2599 2600 2601 2602
}

static int f2fs_set_data_page_dirty(struct page *page)
{
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;

2603 2604
	trace_f2fs_set_page_dirty(page, DATA);

2605 2606
	if (!PageUptodate(page))
		SetPageUptodate(page);
2607

Chao Yu's avatar
Chao Yu committed
2608
	if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
Chao Yu's avatar
Chao Yu committed
2609
		if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
Chao Yu's avatar
Chao Yu committed
2610
			f2fs_register_inmem_page(inode, page);
Chao Yu's avatar
Chao Yu committed
2611 2612 2613 2614 2615 2616 2617
			return 1;
		}
		/*
		 * Previously, this page has been registered, we just
		 * return here.
		 */
		return 0;
2618 2619
	}

2620
	if (!PageDirty(page)) {
2621
		__set_page_dirty_nobuffers(page);
Chao Yu's avatar
Chao Yu committed
2622
		f2fs_update_dirty_page(inode, page);
2623 2624 2625 2626 2627
		return 1;
	}
	return 0;
}

Jaegeuk Kim's avatar
Jaegeuk Kim committed
2628 2629
static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
{
2630 2631
	struct inode *inode = mapping->host;

2632 2633 2634 2635 2636 2637 2638
	if (f2fs_has_inline_data(inode))
		return 0;

	/* make sure allocating whole blocks */
	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
		filemap_write_and_wait(mapping);

2639
	return generic_block_bmap(mapping, block, get_data_block_bmap);
2640 2641
}

2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654
#ifdef CONFIG_MIGRATION
#include <linux/migrate.h>

int f2fs_migrate_page(struct address_space *mapping,
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
	int rc, extra_count;
	struct f2fs_inode_info *fi = F2FS_I(mapping->host);
	bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);

	BUG_ON(PageWriteback(page));

	/* migrating an atomic written page is safe with the inmem_lock hold */
2655 2656 2657 2658 2659 2660
	if (atomic_written) {
		if (mode != MIGRATE_SYNC)
			return -EBUSY;
		if (!mutex_trylock(&fi->inmem_lock))
			return -EAGAIN;
	}
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691

	/*
	 * A reference is expected if PagePrivate set when move mapping,
	 * however F2FS breaks this for maintaining dirty page counts when
	 * truncating pages. So here adjusting the 'extra_count' make it work.
	 */
	extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
	rc = migrate_page_move_mapping(mapping, newpage,
				page, NULL, mode, extra_count);
	if (rc != MIGRATEPAGE_SUCCESS) {
		if (atomic_written)
			mutex_unlock(&fi->inmem_lock);
		return rc;
	}

	if (atomic_written) {
		struct inmem_pages *cur;
		list_for_each_entry(cur, &fi->inmem_pages, list)
			if (cur->page == page) {
				cur->page = newpage;
				break;
			}
		mutex_unlock(&fi->inmem_lock);
		put_page(page);
		get_page(newpage);
	}

	if (PagePrivate(page))
		SetPagePrivate(newpage);
	set_page_private(newpage, page_private(page));

2692 2693 2694 2695
	if (mode != MIGRATE_SYNC_NO_COPY)
		migrate_page_copy(newpage, page);
	else
		migrate_page_states(newpage, page);
2696 2697 2698 2699 2700

	return MIGRATEPAGE_SUCCESS;
}
#endif

2701 2702 2703 2704 2705 2706
const struct address_space_operations f2fs_dblock_aops = {
	.readpage	= f2fs_read_data_page,
	.readpages	= f2fs_read_data_pages,
	.writepage	= f2fs_write_data_page,
	.writepages	= f2fs_write_data_pages,
	.write_begin	= f2fs_write_begin,
2707
	.write_end	= f2fs_write_end,
2708
	.set_page_dirty	= f2fs_set_data_page_dirty,
2709 2710
	.invalidatepage	= f2fs_invalidate_page,
	.releasepage	= f2fs_release_page,
2711
	.direct_IO	= f2fs_direct_IO,
Jaegeuk Kim's avatar
Jaegeuk Kim committed
2712
	.bmap		= f2fs_bmap,
2713 2714 2715
#ifdef CONFIG_MIGRATION
	.migratepage    = f2fs_migrate_page,
#endif
2716
};
2717

Chao Yu's avatar
Chao Yu committed
2718
void f2fs_clear_radix_tree_dirty_tag(struct page *page)
2719 2720 2721 2722 2723 2724 2725 2726 2727 2728
{
	struct address_space *mapping = page_mapping(page);
	unsigned long flags;

	xa_lock_irqsave(&mapping->i_pages, flags);
	radix_tree_tag_clear(&mapping->i_pages, page_index(page),
						PAGECACHE_TAG_DIRTY);
	xa_unlock_irqrestore(&mapping->i_pages, flags);
}

2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
int __init f2fs_init_post_read_processing(void)
{
	bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
	if (!bio_post_read_ctx_cache)
		goto fail;
	bio_post_read_ctx_pool =
		mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
					 bio_post_read_ctx_cache);
	if (!bio_post_read_ctx_pool)
		goto fail_free_cache;
	return 0;

fail_free_cache:
	kmem_cache_destroy(bio_post_read_ctx_cache);
fail:
	return -ENOMEM;
}

void __exit f2fs_destroy_post_read_processing(void)
{
	mempool_destroy(bio_post_read_ctx_pool);
	kmem_cache_destroy(bio_post_read_ctx_cache);
}