buffer.c 66.4 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3
/*
 *  linux/fs/buffer.c
 *
4
 *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
Linus Torvalds's avatar
Linus Torvalds committed
5 6 7
 */

/*
8 9 10
 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
 *
 * Removed a lot of unnecessary code and simplified things now that
Linus Torvalds's avatar
Linus Torvalds committed
11
 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 13 14 15 16 17 18
 *
 * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 *
 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 *
 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
Linus Torvalds's avatar
Linus Torvalds committed
19 20 21
 */

#include <linux/config.h>
Andrew Morton's avatar
Andrew Morton committed
22
#include <linux/kernel.h>
Linus Torvalds's avatar
Linus Torvalds committed
23
#include <linux/fs.h>
24
#include <linux/mm.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28 29 30
#include <linux/smp_lock.h>
#include <linux/blkdev.h>
#include <linux/file.h>
#include <linux/quotaops.h>
#include <linux/iobuf.h>
Linus Torvalds's avatar
Linus Torvalds committed
31
#include <linux/module.h>
32
#include <linux/writeback.h>
33
#include <linux/mempool.h>
Andrew Morton's avatar
Andrew Morton committed
34
#include <linux/hash.h>
Pavel Machek's avatar
Pavel Machek committed
35
#include <linux/suspend.h>
36
#include <linux/buffer_head.h>
Linus Torvalds's avatar
Linus Torvalds committed
37 38
#include <asm/bitops.h>

39 40
static void invalidate_bh_lrus(void);

41
#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
Linus Torvalds's avatar
Linus Torvalds committed
42

Andrew Morton's avatar
Andrew Morton committed
43 44 45 46 47 48 49 50
/*
 * Hashed waitqueue_head's for wait_on_buffer()
 */
#define BH_WAIT_TABLE_ORDER	7
static struct bh_wait_queue_head {
	wait_queue_head_t wqh;
} ____cacheline_aligned_in_smp bh_wait_queue_heads[1<<BH_WAIT_TABLE_ORDER];

51 52
/*
 * Debug/devel support stuff
Linus Torvalds's avatar
Linus Torvalds committed
53
 */
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78

void __buffer_error(char *file, int line)
{
	static int enough;

	if (enough > 10)
		return;
	enough++;
	printk("buffer layer error at %s:%d\n", file, line);
#ifdef CONFIG_X86
	printk("Pass this trace through ksymoops for reporting\n");
	{
		extern void show_stack(long *esp);
		show_stack(0);
	}
#endif
}
EXPORT_SYMBOL(__buffer_error);

inline void
init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
{
	bh->b_end_io = handler;
	bh->b_private = private;
}
Linus Torvalds's avatar
Linus Torvalds committed
79

Andrew Morton's avatar
Andrew Morton committed
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
/*
 * Return the address of the waitqueue_head to be used for this
 * buffer_head
 */
static wait_queue_head_t *bh_waitq_head(struct buffer_head *bh)
{
	return &bh_wait_queue_heads[hash_ptr(bh, BH_WAIT_TABLE_ORDER)].wqh;
}

/*
 * Wait on a buffer until someone does a wakeup on it.  Needs
 * lots of external locking.  ext3 uses this.  Fix it.
 */
void sleep_on_buffer(struct buffer_head *bh)
{
	wait_queue_head_t *wq = bh_waitq_head(bh);
	sleep_on(wq);
}
EXPORT_SYMBOL(sleep_on_buffer);

void wake_up_buffer(struct buffer_head *bh)
{
	wait_queue_head_t *wq = bh_waitq_head(bh);

	if (waitqueue_active(wq))
		wake_up_all(wq);
}
EXPORT_SYMBOL(wake_up_buffer);

Linus Torvalds's avatar
Linus Torvalds committed
109
void unlock_buffer(struct buffer_head *bh)
Linus Torvalds's avatar
Linus Torvalds committed
110
{
111 112 113 114 115 116
	/*
	 * unlock_buffer against a zero-count bh is a bug, if the page
	 * is not locked.  Because then nothing protects the buffer's
	 * waitqueue, which is used here. (Well.  Other locked buffers
	 * against the page will pin it.  But complain anyway).
	 */
117 118 119
	if (atomic_read(&bh->b_count) == 0 &&
			!PageLocked(bh->b_page) &&
			!PageWriteback(bh->b_page))
120 121
		buffer_error();

Andrew Morton's avatar
Andrew Morton committed
122
	clear_buffer_locked(bh);
Linus Torvalds's avatar
Linus Torvalds committed
123
	smp_mb__after_clear_bit();
Andrew Morton's avatar
Andrew Morton committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
	wake_up_buffer(bh);
}

/*
 * Block until a buffer comes unlocked.  This doesn't stop it
 * from becoming locked again - you have to lock it yourself
 * if you want to preserve its state.
 */
void __wait_on_buffer(struct buffer_head * bh)
{
	wait_queue_head_t *wq = bh_waitq_head(bh);
	struct task_struct *tsk = current;
	DECLARE_WAITQUEUE(wait, tsk);

	get_bh(bh);
	add_wait_queue(wq, &wait);
	do {
Jens Axboe's avatar
Jens Axboe committed
141
		blk_run_queues();
Andrew Morton's avatar
Andrew Morton committed
142 143 144 145 146 147 148 149
		set_task_state(tsk, TASK_UNINTERRUPTIBLE);
		if (!buffer_locked(bh))
			break;
		schedule();
	} while (buffer_locked(bh));
	tsk->state = TASK_RUNNING;
	remove_wait_queue(wq, &wait);
	put_bh(bh);
Linus Torvalds's avatar
Linus Torvalds committed
150 151
}

152 153 154 155 156 157
static inline void
__set_page_buffers(struct page *page, struct buffer_head *head)
{
	if (page_has_buffers(page))
		buffer_error();
	page_cache_get(page);
158 159
	SetPagePrivate(page);
	page->private = (unsigned long)head;
160 161 162 163 164
}

static inline void
__clear_page_buffers(struct page *page)
{
165 166
	ClearPagePrivate(page);
	page->private = 0;
167 168 169
	page_cache_release(page);
}

170 171
static void buffer_io_error(struct buffer_head *bh)
{
172 173 174
	printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
			bdevname(bh->b_bdev),
			(unsigned long long)bh->b_blocknr);
175 176
}

Linus Torvalds's avatar
Linus Torvalds committed
177 178 179 180 181
/*
 * Default synchronous end-of-IO handler..  Just mark it up-to-date and
 * unlock the buffer. This is what ll_rw_block uses too.
 */
void end_buffer_io_sync(struct buffer_head *bh, int uptodate)
Linus Torvalds's avatar
Linus Torvalds committed
182
{
183
	if (uptodate) {
Andrew Morton's avatar
Andrew Morton committed
184
		set_buffer_uptodate(bh);
185 186
	} else {
		buffer_io_error(bh);
Andrew Morton's avatar
Andrew Morton committed
187
		clear_buffer_uptodate(bh);
188
	}
Linus Torvalds's avatar
Linus Torvalds committed
189
	unlock_buffer(bh);
Linus Torvalds's avatar
Linus Torvalds committed
190
	put_bh(bh);
Linus Torvalds's avatar
Linus Torvalds committed
191
}
Linus Torvalds's avatar
Linus Torvalds committed
192

Linus Torvalds's avatar
Linus Torvalds committed
193
/*
194 195
 * Write out and wait upon all the dirty data associated with a block
 * device via its mapping.  Does not take the superblock lock.
Linus Torvalds's avatar
Linus Torvalds committed
196
 */
197
int sync_blockdev(struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
198
{
199
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
200

201
	if (bdev) {
202
		int err;
Linus Torvalds's avatar
Linus Torvalds committed
203

204
		ret = filemap_fdatawrite(bdev->bd_inode->i_mapping);
205 206 207
		err = filemap_fdatawait(bdev->bd_inode->i_mapping);
		if (!ret)
			ret = err;
Linus Torvalds's avatar
Linus Torvalds committed
208
	}
209
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
210
}
211
EXPORT_SYMBOL(sync_blockdev);
Linus Torvalds's avatar
Linus Torvalds committed
212

213 214 215 216
/*
 * Write out and wait upon all dirty data associated with this
 * superblock.  Filesystem data as well as the underlying block
 * device.  Takes the superblock lock.
Linus Torvalds's avatar
Linus Torvalds committed
217
 */
Linus Torvalds's avatar
Linus Torvalds committed
218 219
int fsync_super(struct super_block *sb)
{
220
	sync_inodes_sb(sb, 0);
Linus Torvalds's avatar
Linus Torvalds committed
221
	DQUOT_SYNC(sb);
Linus Torvalds's avatar
Linus Torvalds committed
222 223 224 225
	lock_super(sb);
	if (sb->s_dirt && sb->s_op && sb->s_op->write_super)
		sb->s_op->write_super(sb);
	unlock_super(sb);
226 227
	sync_blockdev(sb->s_bdev);
	sync_inodes_sb(sb, 1);
Linus Torvalds's avatar
Linus Torvalds committed
228

229
	return sync_blockdev(sb->s_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
230 231
}

232 233 234 235 236 237
/*
 * Write out and wait upon all dirty data associated with this
 * device.   Filesystem data as well as the underlying block
 * device.  Takes the superblock lock.
 */
int fsync_bdev(struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
238
{
Alexander Viro's avatar
Alexander Viro committed
239
	struct super_block *sb = get_super(bdev);
240 241 242 243 244
	if (sb) {
		int res = fsync_super(sb);
		drop_super(sb);
		return res;
	}
245
	return sync_blockdev(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
246 247
}

248 249 250
/*
 * sync everything.
 */
Linus Torvalds's avatar
Linus Torvalds committed
251 252
asmlinkage long sys_sync(void)
{
253
	sync_inodes(0);	/* All mappings and inodes, including block devices */
254
	DQUOT_SYNC(NULL);
255
	sync_supers();	/* Write the superblocks */
256
	sync_inodes(1);	/* All the mappings and inodes, again. */
Linus Torvalds's avatar
Linus Torvalds committed
257 258 259 260
	return 0;
}

/*
261 262 263
 * Generic function to fsync a file.
 *
 * filp may be NULL if called via the msync of a vma.
Linus Torvalds's avatar
Linus Torvalds committed
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
 */
 
int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
{
	struct inode * inode = dentry->d_inode;
	struct super_block * sb;
	int ret;

	/* sync the inode to buffers */
	write_inode_now(inode, 0);

	/* sync the superblock to buffers */
	sb = inode->i_sb;
	lock_super(sb);
	if (sb->s_op && sb->s_op->write_super)
		sb->s_op->write_super(sb);
	unlock_super(sb);

	/* .. finally sync the buffers to disk */
283
	ret = sync_blockdev(sb->s_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286 287 288 289 290 291
	return ret;
}

asmlinkage long sys_fsync(unsigned int fd)
{
	struct file * file;
	struct dentry * dentry;
	struct inode * inode;
Andrew Morton's avatar
Andrew Morton committed
292
	int ret, err;
Linus Torvalds's avatar
Linus Torvalds committed
293

Andrew Morton's avatar
Andrew Morton committed
294
	ret = -EBADF;
Linus Torvalds's avatar
Linus Torvalds committed
295 296 297 298 299 300 301
	file = fget(fd);
	if (!file)
		goto out;

	dentry = file->f_dentry;
	inode = dentry->d_inode;

Andrew Morton's avatar
Andrew Morton committed
302 303
	ret = -EINVAL;
	if (!file->f_op || !file->f_op->fsync) {
304
		/* Why?  We can still call filemap_fdatawrite */
Linus Torvalds's avatar
Linus Torvalds committed
305
		goto out_putf;
Andrew Morton's avatar
Andrew Morton committed
306
	}
Linus Torvalds's avatar
Linus Torvalds committed
307 308 309

	/* We need to protect against concurrent writers.. */
	down(&inode->i_sem);
310 311 312 313
	ret = filemap_fdatawait(inode->i_mapping);
	err = filemap_fdatawrite(inode->i_mapping);
	if (!ret)
		ret = err;
Linus Torvalds's avatar
Linus Torvalds committed
314
	err = file->f_op->fsync(file, dentry, 0);
315
	if (!ret)
Andrew Morton's avatar
Andrew Morton committed
316 317
		ret = err;
	err = filemap_fdatawait(inode->i_mapping);
318
	if (!ret)
Andrew Morton's avatar
Andrew Morton committed
319
		ret = err;
Linus Torvalds's avatar
Linus Torvalds committed
320 321 322 323 324
	up(&inode->i_sem);

out_putf:
	fput(file);
out:
Andrew Morton's avatar
Andrew Morton committed
325
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
326 327 328 329 330 331 332
}

asmlinkage long sys_fdatasync(unsigned int fd)
{
	struct file * file;
	struct dentry * dentry;
	struct inode * inode;
Andrew Morton's avatar
Andrew Morton committed
333
	int ret, err;
Linus Torvalds's avatar
Linus Torvalds committed
334

Andrew Morton's avatar
Andrew Morton committed
335
	ret = -EBADF;
Linus Torvalds's avatar
Linus Torvalds committed
336 337 338 339 340 341 342
	file = fget(fd);
	if (!file)
		goto out;

	dentry = file->f_dentry;
	inode = dentry->d_inode;

Andrew Morton's avatar
Andrew Morton committed
343
	ret = -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
344 345 346 347
	if (!file->f_op || !file->f_op->fsync)
		goto out_putf;

	down(&inode->i_sem);
348 349 350 351
	ret = filemap_fdatawait(inode->i_mapping);
	err = filemap_fdatawrite(inode->i_mapping);
	if (!ret)
		ret = err;
Linus Torvalds's avatar
Linus Torvalds committed
352
	err = file->f_op->fsync(file, dentry, 1);
353
	if (!ret)
Andrew Morton's avatar
Andrew Morton committed
354 355
		ret = err;
	err = filemap_fdatawait(inode->i_mapping);
356
	if (!ret)
Andrew Morton's avatar
Andrew Morton committed
357
		ret = err;
Linus Torvalds's avatar
Linus Torvalds committed
358 359 360 361 362
	up(&inode->i_sem);

out_putf:
	fput(file);
out:
Andrew Morton's avatar
Andrew Morton committed
363
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
364 365
}

366
/*
367
 * Various filesystems appear to want __find_get_block to be non-blocking.
368
 * But it's the page lock which protects the buffers.  To get around this,
369 370
 * we get exclusion from try_to_free_buffers with the blockdev mapping's
 * private_lock.
371 372 373
 *
 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
 * may be quite high.  This code could TryLock the page, and if that
374 375
 * succeeds, there is no need to take private_lock. (But if
 * private_lock is contended then so is mapping->page_lock).
Linus Torvalds's avatar
Linus Torvalds committed
376
 */
377
struct buffer_head *
378
__find_get_block_slow(struct block_device *bdev, sector_t block, int unused)
Linus Torvalds's avatar
Linus Torvalds committed
379
{
380 381
	struct inode *bd_inode = bdev->bd_inode;
	struct address_space *bd_mapping = bd_inode->i_mapping;
382 383 384 385 386
	struct buffer_head *ret = NULL;
	unsigned long index;
	struct buffer_head *bh;
	struct buffer_head *head;
	struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
387

388 389
	index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
	page = find_get_page(bd_mapping, index);
390 391
	if (!page)
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
392

393
	spin_lock(&bd_mapping->private_lock);
394 395 396 397 398 399 400 401 402
	if (!page_has_buffers(page))
		goto out_unlock;
	head = page_buffers(page);
	bh = head;
	do {
		if (bh->b_blocknr == block) {
			ret = bh;
			get_bh(bh);
			goto out_unlock;
Linus Torvalds's avatar
Linus Torvalds committed
403
		}
404 405 406 407
		bh = bh->b_this_page;
	} while (bh != head);
	buffer_error();
out_unlock:
408
	spin_unlock(&bd_mapping->private_lock);
409 410 411
	page_cache_release(page);
out:
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
412 413 414 415 416 417 418
}

/* If invalidate_buffers() will trash dirty buffers, it means some kind
   of fs corruption is going on. Trashing dirty data always imply losing
   information that was supposed to be just stored on the physical layer
   by the user.

Linus Torvalds's avatar
Linus Torvalds committed
419 420 421 422 423 424
   Thus invalidate_buffers in general usage is not allwowed to trash
   dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
   be preserved.  These buffers are simply skipped.
  
   We also skip buffers which are still in use.  For example this can
   happen if a userspace program is reading the block device.
Linus Torvalds's avatar
Linus Torvalds committed
425 426 427 428 429 430 431 432 433 434 435

   NOTE: In the case where the user removed a removable-media-disk even if
   there's still dirty data not synced on disk (due a bug in the device driver
   or due an error of the user), by not destroying the dirty buffers we could
   generate corruption also on the next media inserted, thus a parameter is
   necessary to handle this case in the most safe way possible (trying
   to not corrupt also the new disk inserted with the data belonging to
   the old now corrupted disk). Also for the ramdisk the natural thing
   to do in order to release the ramdisk memory is to destroy dirty buffers.

   These are two special cases. Normal usage imply the device driver
Linus Torvalds's avatar
Linus Torvalds committed
436
   to issue a sync on the device (without waiting I/O completion) and
Linus Torvalds's avatar
Linus Torvalds committed
437 438 439 440 441 442 443 444 445
   then an invalidate_buffers call that doesn't trash dirty buffers.

   For handling cache coherency with the blkdev pagecache the 'update' case
   is been introduced. It is needed to re-read from disk any pinned
   buffer. NOTE: re-reading from disk is destructive so we can do it only
   when we assume nobody is changing the buffercache under our I/O and when
   we think the disk contains more recent information than the buffercache.
   The update == 1 pass marks the buffers we need to update, the update == 2
   pass does the actual I/O. */
Linus Torvalds's avatar
Linus Torvalds committed
446
void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers)
Linus Torvalds's avatar
Linus Torvalds committed
447
{
448
	invalidate_bh_lrus();
449 450 451 452 453
	/*
	 * FIXME: what about destroy_dirty_buffers?
	 * We really want to use invalidate_inode_pages2() for
	 * that, but not until that's cleaned up.
	 */
454
	current->flags |= PF_INVALIDATE;
Linus Torvalds's avatar
Linus Torvalds committed
455
	invalidate_inode_pages(bdev->bd_inode);
456
	current->flags &= ~PF_INVALIDATE;
Linus Torvalds's avatar
Linus Torvalds committed
457 458
}

Linus Torvalds's avatar
Linus Torvalds committed
459
void __invalidate_buffers(kdev_t dev, int destroy_dirty_buffers)
Linus Torvalds's avatar
Linus Torvalds committed
460
{
Linus Torvalds's avatar
Linus Torvalds committed
461
	struct block_device *bdev = bdget(kdev_t_to_nr(dev));
Linus Torvalds's avatar
Linus Torvalds committed
462 463 464
	if (bdev) {
		invalidate_bdev(bdev, destroy_dirty_buffers);
		bdput(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
465 466 467
	}
}

468 469 470 471
/*
 * FIXME: What is this function actually trying to do?  Why "zones[0]"?
 * Is it still correct/needed if/when blockdev mappings use GFP_HIGHUSER?
 */
Linus Torvalds's avatar
Linus Torvalds committed
472 473
static void free_more_memory(void)
{
474 475 476 477
	zone_t *zone;

	zone = contig_page_data.node_zonelists[GFP_NOFS & GFP_ZONEMASK].zones[0];

Linus Torvalds's avatar
Linus Torvalds committed
478
	wakeup_bdflush();
Linus Torvalds's avatar
Linus Torvalds committed
479
	try_to_free_pages(zone, GFP_NOFS, 0);
Jens Axboe's avatar
Jens Axboe committed
480
	blk_run_queues();
Linus Torvalds's avatar
Linus Torvalds committed
481
	yield();
Linus Torvalds's avatar
Linus Torvalds committed
482 483
}

484
/*
485
 * I/O completion handler for block_read_full_page() - pages
486 487 488
 * which come unlocked at the end of I/O.
 */
static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
Linus Torvalds's avatar
Linus Torvalds committed
489 490 491 492 493
{
	static spinlock_t page_uptodate_lock = SPIN_LOCK_UNLOCKED;
	unsigned long flags;
	struct buffer_head *tmp;
	struct page *page;
494
	int page_uptodate = 1;
Linus Torvalds's avatar
Linus Torvalds committed
495

496 497 498 499
	BUG_ON(!buffer_async_read(bh));

	page = bh->b_page;
	if (uptodate) {
Andrew Morton's avatar
Andrew Morton committed
500
		set_buffer_uptodate(bh);
501
	} else {
Andrew Morton's avatar
Andrew Morton committed
502
		clear_buffer_uptodate(bh);
503
		buffer_io_error(bh);
Linus Torvalds's avatar
Linus Torvalds committed
504
		SetPageError(page);
505
	}
Linus Torvalds's avatar
Linus Torvalds committed
506 507 508 509 510 511 512

	/*
	 * Be _very_ careful from here on. Bad things can happen if
	 * two buffer heads end IO at almost the same time and both
	 * decide that the page is now completely done.
	 */
	spin_lock_irqsave(&page_uptodate_lock, flags);
513
	clear_buffer_async_read(bh);
Linus Torvalds's avatar
Linus Torvalds committed
514
	unlock_buffer(bh);
515 516 517 518
	tmp = bh;
	do {
		if (!buffer_uptodate(tmp))
			page_uptodate = 0;
519
		if (buffer_async_read(tmp)) {
520
			if (buffer_locked(tmp))
521
				goto still_busy;
522 523 524
			if (!buffer_mapped(bh))
				BUG();
		}
Linus Torvalds's avatar
Linus Torvalds committed
525
		tmp = tmp->b_this_page;
526
	} while (tmp != bh);
Linus Torvalds's avatar
Linus Torvalds committed
527 528 529
	spin_unlock_irqrestore(&page_uptodate_lock, flags);

	/*
530 531
	 * If none of the buffers had errors and they are all
	 * uptodate then we can set the page uptodate.
Linus Torvalds's avatar
Linus Torvalds committed
532
	 */
533
	if (page_uptodate && !PageError(page))
Linus Torvalds's avatar
Linus Torvalds committed
534
		SetPageUptodate(page);
535 536 537 538 539 540 541 542 543
	unlock_page(page);
	return;

still_busy:
	spin_unlock_irqrestore(&page_uptodate_lock, flags);
	return;
}

/*
544 545
 * Completion handler for block_write_full_page() - pages which are unlocked
 * during I/O, and which have PageWriteback cleared upon I/O completion.
546 547 548 549 550 551 552 553 554 555 556 557 558
 */
static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
{
	static spinlock_t page_uptodate_lock = SPIN_LOCK_UNLOCKED;
	unsigned long flags;
	struct buffer_head *tmp;
	struct page *page;

	BUG_ON(!buffer_async_write(bh));

	page = bh->b_page;
	if (uptodate) {
		set_buffer_uptodate(bh);
559
	} else {
560
		buffer_io_error(bh);
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
		clear_buffer_uptodate(bh);
		SetPageError(page);
	}

	spin_lock_irqsave(&page_uptodate_lock, flags);
	clear_buffer_async_write(bh);
	unlock_buffer(bh);
	tmp = bh->b_this_page;
	while (tmp != bh) {
		if (buffer_async_write(tmp)) {
			if (buffer_locked(tmp))
				goto still_busy;
			if (!buffer_mapped(bh))
				BUG();
		}
		tmp = tmp->b_this_page;
577
	}
578 579
	spin_unlock_irqrestore(&page_uptodate_lock, flags);
	end_page_writeback(page);
Linus Torvalds's avatar
Linus Torvalds committed
580 581 582 583 584 585 586
	return;

still_busy:
	spin_unlock_irqrestore(&page_uptodate_lock, flags);
	return;
}

587
/*
588
 * If a page's buffers are under async readin (end_buffer_async_read
589 590 591
 * completion) then there is a possibility that another thread of
 * control could lock one of the buffers after it has completed
 * but while some of the other buffers have not completed.  This
592 593
 * locked buffer would confuse end_buffer_async_read() into not unlocking
 * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
594 595 596 597 598
 * that this buffer is not under async I/O.
 *
 * The page comes unlocked when it has no locked buffer_async buffers
 * left.
 *
599
 * PageLocked prevents anyone starting new async I/O reads any of
600
 * the buffers.
601 602 603 604 605 606
 *
 * PageWriteback is used to prevent simultaneous writeout of the same
 * page.
 *
 * PageLocked prevents anyone from starting writeback of a page which is
 * under read I/O (PageWriteback is only ever set against a locked page).
607
 */
608
inline void mark_buffer_async_read(struct buffer_head *bh)
Linus Torvalds's avatar
Linus Torvalds committed
609
{
610 611
	bh->b_end_io = end_buffer_async_read;
	set_buffer_async_read(bh);
Linus Torvalds's avatar
Linus Torvalds committed
612
}
613 614 615 616 617 618 619 620
EXPORT_SYMBOL(mark_buffer_async_read);

inline void mark_buffer_async_write(struct buffer_head *bh)
{
	bh->b_end_io = end_buffer_async_write;
	set_buffer_async_write(bh);
}
EXPORT_SYMBOL(mark_buffer_async_write);
Linus Torvalds's avatar
Linus Torvalds committed
621

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693

/*
 * fs/buffer.c contains helper functions for buffer-backed address space's
 * fsync functions.  A common requirement for buffer-based filesystems is
 * that certain data from the backing blockdev needs to be written out for
 * a successful fsync().  For example, ext2 indirect blocks need to be
 * written back and waited upon before fsync() returns.
 *
 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
 * management of a list of dependent buffers at ->i_mapping->private_list.
 *
 * Locking is a little subtle: try_to_free_buffers() will remove buffers
 * from their controlling inode's queue when they are being freed.  But
 * try_to_free_buffers() will be operating against the *blockdev* mapping
 * at the time, not against the S_ISREG file which depends on those buffers.
 * So the locking for private_list is via the private_lock in the address_space
 * which backs the buffers.  Which is different from the address_space 
 * against which the buffers are listed.  So for a particular address_space,
 * mapping->private_lock does *not* protect mapping->private_list!  In fact,
 * mapping->private_list will always be protected by the backing blockdev's
 * ->private_lock.
 *
 * Which introduces a requirement: all buffers on an address_space's
 * ->private_list must be from the same address_space: the blockdev's.
 *
 * address_spaces which do not place buffers at ->private_list via these
 * utility functions are free to use private_lock and private_list for
 * whatever they want.  The only requirement is that list_empty(private_list)
 * be true at clear_inode() time.
 *
 * FIXME: clear_inode should not call invalidate_inode_buffers().  The
 * filesystems should do that.  invalidate_inode_buffers() should just go
 * BUG_ON(!list_empty).
 *
 * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
 * take an address_space, not an inode.  And it should be called
 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
 * queued up.
 *
 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
 * list if it is already on a list.  Because if the buffer is on a list,
 * it *must* already be on the right one.  If not, the filesystem is being
 * silly.  This will save a ton of locking.  But first we have to ensure
 * that buffers are taken *off* the old inode's list when they are freed
 * (presumably in truncate).  That requires careful auditing of all
 * filesystems (do it inside bforget()).  It could also be done by bringing
 * b_inode back.
 */

void buffer_insert_list(spinlock_t *lock,
		struct buffer_head *bh, struct list_head *list)
{
	spin_lock(lock);
	list_del(&bh->b_assoc_buffers);
	list_add(&bh->b_assoc_buffers, list);
	spin_unlock(lock);
}

/*
 * The buffer's backing address_space's private_lock must be held
 */
static inline void __remove_assoc_queue(struct buffer_head *bh)
{
	list_del_init(&bh->b_assoc_buffers);
}

int inode_has_buffers(struct inode *inode)
{
	return !list_empty(&inode->i_mapping->private_list);
}

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
/*
 * osync is designed to support O_SYNC io.  It waits synchronously for
 * all already-submitted IO to complete, but does not queue any new
 * writes to the disk.
 *
 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
 * you dirty the buffers, and then use osync_inode_buffers to wait for
 * completion.  Any other dirty buffers which are not yet queued for
 * write will not be flushed to disk by the osync.
 */
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
{
	struct buffer_head *bh;
	struct list_head *p;
	int err = 0;

	spin_lock(lock);
repeat:
712 713
	list_for_each_prev(p, list) {
		bh = BH_ENTRY(p);
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
		if (buffer_locked(bh)) {
			get_bh(bh);
			spin_unlock(lock);
			wait_on_buffer(bh);
			if (!buffer_uptodate(bh))
				err = -EIO;
			brelse(bh);
			spin_lock(lock);
			goto repeat;
		}
	}
	spin_unlock(lock);
	return err;
}

729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
/**
 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
 *                        buffers
 * @buffer_mapping - the mapping which backs the buffers' data
 * @mapping - the mapping which wants those buffers written
 *
 * Starts I/O against the buffers at mapping->private_list, and waits upon
 * that I/O.
 *
 * Basically, this is a convenience function for fsync().  @buffer_mapping is
 * the blockdev which "owns" the buffers and @mapping is a file or directory
 * which needs those buffers to be written for a successful fsync().
 */
int sync_mapping_buffers(struct address_space *mapping)
{
	struct address_space *buffer_mapping = mapping->assoc_mapping;

	if (buffer_mapping == NULL || list_empty(&mapping->private_list))
		return 0;

	return fsync_buffers_list(&buffer_mapping->private_lock,
					&mapping->private_list);
}
EXPORT_SYMBOL(sync_mapping_buffers);

754 755 756 757 758 759 760 761 762
/**
 * write_mapping_buffers - Start writeout of a mapping's "associated" buffers.
 * @mapping - the mapping which wants those buffers written.
 *
 * Starts I/O against dirty buffers which are on @mapping->private_list.
 * Those buffers must be backed by @mapping->assoc_mapping.
 *
 * The private_list buffers generally contain filesystem indirect blocks.
 * The idea is that the filesystem can start I/O against the indirects at
763
 * the same time as running generic_writepages(), so the indirect's
764 765 766
 * I/O will be merged with the data.
 *
 * We sneakliy write the buffers in probable tail-to-head order.  This is
767
 * because generic_writepages() writes in probable head-to-tail
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
 * order.  If the file is so huge that the data or the indirects overflow
 * the request queue we will at least get some merging this way.
 *
 * Any clean+unlocked buffers are de-listed.  clean/locked buffers must be
 * left on the list for an fsync() to wait on.
 *
 * Couldn't think of a smart way of avoiding livelock, so chose the dumb
 * way instead.
 *
 * FIXME: duplicates fsync_inode_buffers() functionality a bit.
 */
int write_mapping_buffers(struct address_space *mapping)
{
	spinlock_t *lock;
	struct address_space *buffer_mapping;
	unsigned nr_to_write;	/* livelock avoidance */
	struct list_head *lh;
	int ret = 0;

	if (list_empty(&mapping->private_list))
		goto out;

	buffer_mapping = mapping->assoc_mapping;
	lock = &buffer_mapping->private_lock;
	spin_lock(lock);
	nr_to_write = 0;
	lh = mapping->private_list.next;
	while (lh != &mapping->private_list) {
		lh = lh->next;
		nr_to_write++;
	}
	nr_to_write *= 2;	/* Allow for some late additions */

	while (nr_to_write-- && !list_empty(&mapping->private_list)) {
		struct buffer_head *bh;

		bh = BH_ENTRY(mapping->private_list.prev);
		list_del_init(&bh->b_assoc_buffers);
		if (!buffer_dirty(bh) && !buffer_locked(bh))
			continue;
		/* Stick it on the far end of the list. Order is preserved. */
		list_add(&bh->b_assoc_buffers, &mapping->private_list);
		if (test_set_buffer_locked(bh))
			continue;
		get_bh(bh);
		spin_unlock(lock);
		if (test_clear_buffer_dirty(bh)) {
			bh->b_end_io = end_buffer_io_sync;
			submit_bh(WRITE, bh);
		} else {
			unlock_buffer(bh);
			put_bh(bh);
		}
		spin_lock(lock);
	}
	spin_unlock(lock);
out:
	return ret;
}
827
EXPORT_SYMBOL(write_mapping_buffers);
828

829 830 831 832 833 834 835 836 837 838 839 840
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
{
	struct address_space *mapping = inode->i_mapping;
	struct address_space *buffer_mapping = bh->b_page->mapping;

	mark_buffer_dirty(bh);
	if (!mapping->assoc_mapping) {
		mapping->assoc_mapping = buffer_mapping;
	} else {
		if (mapping->assoc_mapping != buffer_mapping)
			BUG();
	}
841 842 843
	if (list_empty(&bh->b_assoc_buffers))
		buffer_insert_list(&buffer_mapping->private_lock,
				bh, &mapping->private_list);
844 845 846
}
EXPORT_SYMBOL(mark_buffer_dirty_inode);

Linus Torvalds's avatar
Linus Torvalds committed
847
/*
848
 * Write out and wait upon a list of buffers.
Linus Torvalds's avatar
Linus Torvalds committed
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
 *
 * We have conflicting pressures: we want to make sure that all
 * initially dirty buffers get waited on, but that any subsequently
 * dirtied buffers don't.  After all, we don't want fsync to last
 * forever if somebody is actively writing to the file.
 *
 * Do this in two main stages: first we copy dirty buffers to a
 * temporary inode list, queueing the writes as we go.  Then we clean
 * up, waiting for those writes to complete.
 * 
 * During this second stage, any subsequent updates to the file may end
 * up refiling the buffer on the original inode's dirty list again, so
 * there is a chance we will end up with a buffer queued for write but
 * not yet completed on that list.  So, as a final cleanup we go through
 * the osync code to catch these locked, dirty buffers without requeuing
 * any newly dirty buffers for write.
 */
866
int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
Linus Torvalds's avatar
Linus Torvalds committed
867 868
{
	struct buffer_head *bh;
Linus Torvalds's avatar
Linus Torvalds committed
869
	struct list_head tmp;
Linus Torvalds's avatar
Linus Torvalds committed
870
	int err = 0, err2;
871

Linus Torvalds's avatar
Linus Torvalds committed
872
	INIT_LIST_HEAD(&tmp);
Linus Torvalds's avatar
Linus Torvalds committed
873

874
	spin_lock(lock);
Linus Torvalds's avatar
Linus Torvalds committed
875 876
	while (!list_empty(list)) {
		bh = BH_ENTRY(list->next);
877
		list_del_init(&bh->b_assoc_buffers);
878
		if (buffer_dirty(bh) || buffer_locked(bh)) {
879
			list_add(&bh->b_assoc_buffers, &tmp);
Linus Torvalds's avatar
Linus Torvalds committed
880 881
			if (buffer_dirty(bh)) {
				get_bh(bh);
882
				spin_unlock(lock);
Linus Torvalds's avatar
Linus Torvalds committed
883 884
				ll_rw_block(WRITE, 1, &bh);
				brelse(bh);
885
				spin_lock(lock);
Linus Torvalds's avatar
Linus Torvalds committed
886 887 888 889
			}
		}
	}

Linus Torvalds's avatar
Linus Torvalds committed
890 891
	while (!list_empty(&tmp)) {
		bh = BH_ENTRY(tmp.prev);
892
		__remove_assoc_queue(bh);
Linus Torvalds's avatar
Linus Torvalds committed
893
		get_bh(bh);
894
		spin_unlock(lock);
Linus Torvalds's avatar
Linus Torvalds committed
895 896 897 898
		wait_on_buffer(bh);
		if (!buffer_uptodate(bh))
			err = -EIO;
		brelse(bh);
899
		spin_lock(lock);
Linus Torvalds's avatar
Linus Torvalds committed
900 901
	}
	
902 903
	spin_unlock(lock);
	err2 = osync_buffers_list(lock, list);
Linus Torvalds's avatar
Linus Torvalds committed
904 905 906 907 908
	if (err)
		return err;
	else
		return err2;
}
Linus Torvalds's avatar
Linus Torvalds committed
909 910

/*
911 912 913
 * Invalidate any and all dirty buffers on a given inode.  We are
 * probably unmounting the fs, but that doesn't mean we have already
 * done a sync().  Just drop the buffers from the inode list.
914 915 916 917
 *
 * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
 * assumes that all the buffers are against the blockdev.  Not true
 * for reiserfs.
Linus Torvalds's avatar
Linus Torvalds committed
918
 */
919 920
void invalidate_inode_buffers(struct inode *inode)
{
921 922 923 924 925 926 927 928 929 930
	if (inode_has_buffers(inode)) {
		struct address_space *mapping = inode->i_mapping;
		struct list_head *list = &mapping->private_list;
		struct address_space *buffer_mapping = mapping->assoc_mapping;

		spin_lock(&buffer_mapping->private_lock);
		while (!list_empty(list))
			__remove_assoc_queue(BH_ENTRY(list->next));
		spin_unlock(&buffer_mapping->private_lock);
	}
931
}
Linus Torvalds's avatar
Linus Torvalds committed
932

933 934 935 936 937
/*
 * Create the appropriate buffers when given a page for data area and
 * the size of each buffer.. Use the bh->b_this_page linked list to
 * follow the buffers created.  Return NULL if unable to create more
 * buffers.
938 939 940
 *
 * The retry flag is used to differentiate async IO (paging, swapping)
 * which may not fail from ordinary buffer allocations.
941 942
 */
static struct buffer_head *
943
create_buffers(struct page * page, unsigned long size, int retry)
944 945 946 947 948 949 950 951
{
	struct buffer_head *bh, *head;
	long offset;

try_again:
	head = NULL;
	offset = PAGE_SIZE;
	while ((offset -= size) >= 0) {
952
		current->flags |= PF_NOWARN;
953
		bh = alloc_buffer_head();
954
		current->flags &= ~PF_NOWARN;
955 956 957 958 959
		if (!bh)
			goto no_grow;

		bh->b_bdev = NULL;
		bh->b_this_page = head;
960
		bh->b_blocknr = -1;
961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
		head = bh;

		bh->b_state = 0;
		atomic_set(&bh->b_count, 0);
		bh->b_size = size;

		/* Link the buffer to its page */
		set_bh_page(bh, page, offset);

		bh->b_end_io = NULL;
	}
	return head;
/*
 * In case anything failed, we just free everything we got.
 */
no_grow:
	if (head) {
		do {
			bh = head;
			head = head->b_this_page;
981
			free_buffer_head(bh);
982 983 984 985 986 987 988 989 990
		} while (head);
	}

	/*
	 * Return failure for non-async IO requests.  Async IO requests
	 * are not allowed to fail, so we have to wait until buffer heads
	 * become available.  But we don't want tasks sleeping with 
	 * partially complete buffers, so all were released above.
	 */
991
	if (!retry)
992 993 994 995 996 997 998 999
		return NULL;

	/* We're _really_ low on memory. Now we just
	 * wait for old buffer heads to become free due to
	 * finishing IO.  Since this is an async request and
	 * the reserve list is empty, we're sure there are 
	 * async buffer heads in use.
	 */
Jens Axboe's avatar
Jens Axboe committed
1000
	blk_run_queues();
1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031

	free_more_memory();
	goto try_again;
}

static inline void
link_dev_buffers(struct page *page, struct buffer_head *head)
{
	struct buffer_head *bh, *tail;

	bh = head;
	do {
		tail = bh;
		bh = bh->b_this_page;
	} while (bh);
	tail->b_this_page = head;
	__set_page_buffers(page, head);
}

/*
 * Initialise the state of a blockdev page's buffers.
 */ 
static /*inline*/ void
init_page_buffers(struct page *page, struct block_device *bdev,
			int block, int size)
{
	struct buffer_head *head = page_buffers(page);
	struct buffer_head *bh = head;
	unsigned int b_state;

	b_state = 1 << BH_Mapped;
Andrew Morton's avatar
Andrew Morton committed
1032
	if (PageUptodate(page))
1033 1034 1035 1036 1037 1038 1039 1040
		b_state |= 1 << BH_Uptodate;

	do {
		if (!(bh->b_state & (1 << BH_Mapped))) {
			init_buffer(bh, NULL, NULL);
			bh->b_bdev = bdev;
			bh->b_blocknr = block;
			bh->b_state = b_state;
Linus Torvalds's avatar
Linus Torvalds committed
1041
		}
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
		block++;
		bh = bh->b_this_page;
	} while (bh != head);
}

/*
 * Create the page-cache page that contains the requested block.
 *
 * This is user purely for blockdev mappings.
 */
static /*inline*/ struct page *
grow_dev_page(struct block_device *bdev, unsigned long block,
			unsigned long index, int size)
{
	struct inode *inode = bdev->bd_inode;
	struct page *page;
	struct buffer_head *bh;

	page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
	if (!page)
		return NULL;

	if (!PageLocked(page))
		BUG();

	if (page_has_buffers(page)) {
		bh = page_buffers(page);
		if (bh->b_size == size)
			return page;
		if (!try_to_free_buffers(page))
			goto failed;
Linus Torvalds's avatar
Linus Torvalds committed
1073 1074
	}

1075 1076 1077 1078 1079 1080 1081 1082 1083
	/*
	 * Allocate some buffers for this page
	 */
	bh = create_buffers(page, size, 0);
	if (!bh)
		goto failed;

	/*
	 * Link the page to the buffers and initialise them.  Take the
1084
	 * lock to be atomic wrt __find_get_block(), which does not
1085 1086
	 * run under the page lock.
	 */
1087
	spin_lock(&inode->i_mapping->private_lock);
1088 1089
	link_dev_buffers(page, bh);
	init_page_buffers(page, bdev, block, size);
1090
	spin_unlock(&inode->i_mapping->private_lock);
1091 1092 1093 1094
	return page;

failed:
	buffer_error();
Andrew Morton's avatar
Andrew Morton committed
1095
	unlock_page(page);
1096 1097
	page_cache_release(page);
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1098 1099
}

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
/*
 * Create buffers for the specified block device block's page.  If
 * that page was dirty, the buffers are set dirty also.
 *
 * Except that's a bug.  Attaching dirty buffers to a dirty
 * blockdev's page can result in filesystem corruption, because
 * some of those buffers may be aliases of filesystem data.
 * grow_dev_page() will go BUG() if this happens.
 */
static inline int
grow_buffers(struct block_device *bdev, unsigned long block, int size)
{
	struct page *page;
	unsigned long index;
	int sizebits;

	/* Size must be multiple of hard sectorsize */
	if (size & (bdev_hardsect_size(bdev)-1))
		BUG();
	if (size < 512 || size > PAGE_SIZE)
		BUG();

	sizebits = -1;
	do {
		sizebits++;
	} while ((size << sizebits) < PAGE_SIZE);

	index = block >> sizebits;
	block = index << sizebits;

	/* Create a page with the proper size buffers.. */
	page = grow_dev_page(bdev, block, index, size);
	if (!page)
		return 0;
Andrew Morton's avatar
Andrew Morton committed
1134
	unlock_page(page);
1135 1136
	page_cache_release(page);
	return 1;
Linus Torvalds's avatar
Linus Torvalds committed
1137 1138 1139
}

/*
1140 1141 1142
 * __getblk will locate (and, if necessary, create) the buffer_head
 * which corresponds to the passed block_device, block and size. The
 * returned buffer has its reference count incremented.
Linus Torvalds's avatar
Linus Torvalds committed
1143
 *
1144 1145 1146
 * __getblk() cannot fail - it just keeps trying.  If you pass it an
 * illegal block number, __getblk() will happily return a buffer_head
 * which represents the non-existent block.  Very weird.
Linus Torvalds's avatar
Linus Torvalds committed
1147
 *
1148 1149
 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
 * attempt is failing.  FIXME, perhaps?
Linus Torvalds's avatar
Linus Torvalds committed
1150
 */
1151
struct buffer_head *
1152
__getblk_slow(struct block_device *bdev, sector_t block, int size)
Linus Torvalds's avatar
Linus Torvalds committed
1153
{
Linus Torvalds's avatar
Linus Torvalds committed
1154 1155
	for (;;) {
		struct buffer_head * bh;
Linus Torvalds's avatar
Linus Torvalds committed
1156

1157
		bh = __find_get_block(bdev, block, size);
1158 1159
		if (bh) {
			touch_buffer(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1160
			return bh;
1161
		}
Linus Torvalds's avatar
Linus Torvalds committed
1162

Linus Torvalds's avatar
Linus Torvalds committed
1163
		if (!grow_buffers(bdev, block, size))
Linus Torvalds's avatar
Linus Torvalds committed
1164
			free_more_memory();
Linus Torvalds's avatar
Linus Torvalds committed
1165 1166 1167 1168
	}
}

/*
1169 1170 1171 1172 1173 1174 1175 1176
 * The relationship between dirty buffers and dirty pages:
 *
 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
 * the page appears on its address_space.dirty_pages list.
 *
 * At all times, the dirtiness of the buffers represents the dirtiness of
 * subsections of the page.  If the page has buffers, the page dirty bit is
 * merely a hint about the true dirty state.
Linus Torvalds's avatar
Linus Torvalds committed
1177
 *
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
 * When a page is set dirty in its entirety, all its buffers are marked dirty
 * (if the page has buffers).
 *
 * When a buffer is marked dirty, its page is dirtied, but the page's other
 * buffers are not.
 *
 * Also.  When blockdev buffers are explicitly read with bread(), they
 * individually become uptodate.  But their backing page remains not
 * uptodate - even if all of its buffers are uptodate.  A subsequent
 * block_read_full_page() against that page will discover all the uptodate
 * buffers, will set the page uptodate and will perform no I/O.
Linus Torvalds's avatar
Linus Torvalds committed
1189 1190
 */

1191 1192 1193 1194 1195 1196 1197 1198
/**
 * mark_buffer_dirty - mark a buffer_head as needing writeout
 *
 * mark_buffer_dirty() will set the dirty bit against the buffer,
 * then set its backing page dirty, then attach the page to its
 * address_space's dirty_pages list and then attach the address_space's
 * inode to its superblock's dirty inode list.
 *
1199
 * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1200 1201
 * mapping->page_lock and the global inode_lock.
 */
Linus Torvalds's avatar
Linus Torvalds committed
1202 1203
void mark_buffer_dirty(struct buffer_head *bh)
{
1204 1205
	if (!buffer_uptodate(bh))
		buffer_error();
1206
	if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh))
1207
		__set_page_dirty_nobuffers(bh->b_page);
Linus Torvalds's avatar
Linus Torvalds committed
1208 1209 1210
}

/*
1211 1212 1213 1214 1215
 * Decrement a buffer_head's reference count.  If all buffers against a page
 * have zero reference count, are clean and unlocked, and if the page is clean
 * and unlocked then try_to_free_buffers() may strip the buffers from the page
 * in preparation for freeing it (sometimes, rarely, buffers are removed from
 * a page but it ends up not being freed, and buffers may later be reattached).
Linus Torvalds's avatar
Linus Torvalds committed
1216 1217 1218 1219
 */
void __brelse(struct buffer_head * buf)
{
	if (atomic_read(&buf->b_count)) {
Linus Torvalds's avatar
Linus Torvalds committed
1220
		put_bh(buf);
Linus Torvalds's avatar
Linus Torvalds committed
1221 1222
		return;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1223
	printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1224
	buffer_error();		/* For the stack backtrace */
Linus Torvalds's avatar
Linus Torvalds committed
1225 1226 1227
}

/*
Linus Torvalds's avatar
Linus Torvalds committed
1228
 * bforget() is like brelse(), except it discards any
Linus Torvalds's avatar
Linus Torvalds committed
1229
 * potentially dirty data.
Linus Torvalds's avatar
Linus Torvalds committed
1230
 */
1231
void __bforget(struct buffer_head *bh)
Linus Torvalds's avatar
Linus Torvalds committed
1232
{
1233 1234 1235 1236 1237 1238 1239 1240 1241
	clear_buffer_dirty(bh);
	if (!list_empty(&bh->b_assoc_buffers)) {
		struct address_space *buffer_mapping = bh->b_page->mapping;

		spin_lock(&buffer_mapping->private_lock);
		list_del_init(&bh->b_assoc_buffers);
		spin_unlock(&buffer_mapping->private_lock);
	}
	__brelse(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1242 1243
}

Linus Torvalds's avatar
Linus Torvalds committed
1244
/**
1245
 *  __bread() - reads a specified block and returns the bh
1246 1247
 *  @block: number of block
 *  @size: size (in bytes) to read
Linus Torvalds's avatar
Linus Torvalds committed
1248
 * 
1249 1250
 *  Reads a specified block, and returns buffer head that contains it.
 *  It returns NULL if the block was unreadable.
Linus Torvalds's avatar
Linus Torvalds committed
1251
 */
1252 1253
struct buffer_head *
__bread_slow(struct block_device *bdev, sector_t block, int size)
Linus Torvalds's avatar
Linus Torvalds committed
1254
{
1255
	struct buffer_head *bh = __getblk(bdev, block, size);
Linus Torvalds's avatar
Linus Torvalds committed
1256 1257 1258

	if (buffer_uptodate(bh))
		return bh;
1259 1260 1261
	lock_buffer(bh);
	if (buffer_uptodate(bh)) {
		unlock_buffer(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1262 1263
		return bh;
	} else {
1264 1265 1266 1267 1268 1269 1270 1271
		if (buffer_dirty(bh))
			buffer_error();
		get_bh(bh);
		bh->b_end_io = end_buffer_io_sync;
		submit_bh(READ, bh);
		wait_on_buffer(bh);
		if (buffer_uptodate(bh))
			return bh;
Linus Torvalds's avatar
Linus Torvalds committed
1272
	}
1273 1274
	brelse(bh);
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1275 1276
}

1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
/*
 * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
 * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
 * refcount elevated by one when they're in an LRU.  A buffer can only appear
 * once in a particular CPU's LRU.  A single buffer can be present in multiple
 * CPU's LRUs at the same time.
 *
 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
 * sb_find_get_block().
 */

#define BH_LRU_SIZE	7

static struct bh_lru {
	spinlock_t lock;
	struct buffer_head *bhs[BH_LRU_SIZE];
} ____cacheline_aligned_in_smp bh_lrus[NR_CPUS];

/*
 * The LRU management algorithm is dopey-but-simple.  Sorry.
 */
static void bh_lru_install(struct buffer_head *bh)
{
	struct buffer_head *evictee = NULL;
	struct bh_lru *lru;

	if (bh == NULL)
		return;

	lru = &bh_lrus[get_cpu()];
	spin_lock(&lru->lock);
	if (lru->bhs[0] != bh) {
		struct buffer_head *bhs[BH_LRU_SIZE];
		int in;
		int out = 0;

		get_bh(bh);
		bhs[out++] = bh;
		for (in = 0; in < BH_LRU_SIZE; in++) {
			struct buffer_head *bh2 = lru->bhs[in];

			if (bh2 == bh) {
				__brelse(bh2);
			} else {
				if (out >= BH_LRU_SIZE) {
					BUG_ON(evictee != NULL);
					evictee = bh2;
				} else {
					bhs[out++] = bh2;
				}
			}
		}
		while (out < BH_LRU_SIZE)
			bhs[out++] = NULL;
		memcpy(lru->bhs, bhs, sizeof(bhs));
	}
	spin_unlock(&lru->lock);
	put_cpu();

	if (evictee) {
		touch_buffer(evictee);
		__brelse(evictee);
	}
}

static inline struct buffer_head *
lookup_bh(struct block_device *bdev, sector_t block, int size)
{
	struct buffer_head *ret = NULL;
	struct bh_lru *lru;
	int i;

	lru = &bh_lrus[get_cpu()];
	spin_lock(&lru->lock);
	for (i = 0; i < BH_LRU_SIZE; i++) {
		struct buffer_head *bh = lru->bhs[i];

		if (bh && bh->b_bdev == bdev &&
				bh->b_blocknr == block && bh->b_size == size) {
			if (i) {
				while (i) {
					lru->bhs[i] = lru->bhs[i - 1];
					i--;
				}
				lru->bhs[0] = bh;
			}
			get_bh(bh);
			ret = bh;
			break;
		}
	}
	spin_unlock(&lru->lock);
	put_cpu();
	return ret;
}

struct buffer_head *
__find_get_block(struct block_device *bdev, sector_t block, int size)
{
	struct buffer_head *bh = lookup_bh(bdev, block, size);

	if (bh == NULL) {
		bh = __find_get_block_slow(bdev, block, size);
		bh_lru_install(bh);
	}
	return bh;
}
EXPORT_SYMBOL(__find_get_block);

struct buffer_head *
__getblk(struct block_device *bdev, sector_t block, int size)
{
	struct buffer_head *bh = __find_get_block(bdev, block, size);

	if (bh == NULL) {
		bh = __getblk_slow(bdev, block, size);
		bh_lru_install(bh);
	}
	return bh;
}
EXPORT_SYMBOL(__getblk);

struct buffer_head *
__bread(struct block_device *bdev, sector_t block, int size)
{
	struct buffer_head *bh = __getblk(bdev, block, size);

	if (bh) {
		if (buffer_uptodate(bh))
			return bh;
		__brelse(bh);
	}
	bh = __bread_slow(bdev, block, size);
	bh_lru_install(bh);
	return bh;
}
EXPORT_SYMBOL(__bread);

/*
 * This is called rarely - at unmount.
 */
static void invalidate_bh_lrus(void)
{
	int cpu_idx;

	for (cpu_idx = 0; cpu_idx < NR_CPUS; cpu_idx++)
		spin_lock(&bh_lrus[cpu_idx].lock);
	for (cpu_idx = 0; cpu_idx < NR_CPUS; cpu_idx++) {
		int i;

		for (i = 0; i < BH_LRU_SIZE; i++) {
			brelse(bh_lrus[cpu_idx].bhs[i]);
			bh_lrus[cpu_idx].bhs[i] = NULL;
		}
	}
	for (cpu_idx = 0; cpu_idx < NR_CPUS; cpu_idx++)
		spin_unlock(&bh_lrus[cpu_idx].lock);
}

1436 1437
void set_bh_page(struct buffer_head *bh,
		struct page *page, unsigned long offset)
Linus Torvalds's avatar
Linus Torvalds committed
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
{
	bh->b_page = page;
	if (offset >= PAGE_SIZE)
		BUG();
	if (PageHighMem(page))
		/*
		 * This catches illegal uses and preserves the offset:
		 */
		bh->b_data = (char *)(0 + offset);
	else
		bh->b_data = page_address(page) + offset;
}
Linus Torvalds's avatar
Linus Torvalds committed
1450
EXPORT_SYMBOL(set_bh_page);
Linus Torvalds's avatar
Linus Torvalds committed
1451

Linus Torvalds's avatar
Linus Torvalds committed
1452 1453 1454
/*
 * Called when truncating a buffer on a page completely.
 */
1455
static /* inline */ void discard_buffer(struct buffer_head * bh)
Linus Torvalds's avatar
Linus Torvalds committed
1456
{
1457 1458 1459 1460 1461 1462 1463
	lock_buffer(bh);
	clear_buffer_dirty(bh);
	bh->b_bdev = NULL;
	clear_buffer_mapped(bh);
	clear_buffer_req(bh);
	clear_buffer_new(bh);
	unlock_buffer(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1464 1465
}

Linus Torvalds's avatar
Linus Torvalds committed
1466
/**
1467 1468 1469 1470
 * try_to_release_page() - release old fs-specific metadata on a page
 *
 * @page: the page which the kernel is trying to free
 * @gfp_mask: memory allocation flags (and I/O mode)
Linus Torvalds's avatar
Linus Torvalds committed
1471
 *
1472 1473 1474 1475 1476 1477 1478 1479
 * The address_space is to try to release any data against the page
 * (presumably at page->private).  If the release was successful, return `1'.
 * Otherwise return zero.
 *
 * The @gfp_mask argument specifies whether I/O may be performed to release
 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
 *
 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
Linus Torvalds's avatar
Linus Torvalds committed
1480
 */
1481
int try_to_release_page(struct page *page, int gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1482
{
1483 1484
	struct address_space * const mapping = page->mapping;

Linus Torvalds's avatar
Linus Torvalds committed
1485 1486
	if (!PageLocked(page))
		BUG();
1487 1488
	if (PageWriteback(page))
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1489
	
1490 1491
	if (mapping && mapping->a_ops->releasepage)
		return mapping->a_ops->releasepage(page, gfp_mask);
1492
	return try_to_free_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
1493 1494
}

1495
/**
1496
 * block_invalidatepage - invalidate part of all of a buffer-backed page
1497 1498 1499 1500
 *
 * @page: the page which is affected
 * @offset: the index of the truncation point
 *
1501 1502
 * block_invalidatepage() is called when all or part of the page has become
 * invalidatedby a truncate operation.
1503
 *
1504
 * block_invalidatepage() does not have to release all buffers, but it must
1505 1506 1507 1508
 * ensure that no dirty buffer is left outside @offset and that no I/O
 * is underway against any of the blocks which are outside the truncation
 * point.  Because the caller is about to free (and possibly reuse) those
 * blocks on-disk.
Linus Torvalds's avatar
Linus Torvalds committed
1509
 */
1510
int block_invalidatepage(struct page *page, unsigned long offset)
Linus Torvalds's avatar
Linus Torvalds committed
1511 1512 1513
{
	struct buffer_head *head, *bh, *next;
	unsigned int curr_off = 0;
1514
	int ret = 1;
Linus Torvalds's avatar
Linus Torvalds committed
1515

1516
	BUG_ON(!PageLocked(page));
1517
	if (!page_has_buffers(page))
1518
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
1519

1520
	head = page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
1521 1522 1523 1524 1525 1526
	bh = head;
	do {
		unsigned int next_off = curr_off + bh->b_size;
		next = bh->b_this_page;

		/*
1527
		 * is this block fully invalidated?
Linus Torvalds's avatar
Linus Torvalds committed
1528 1529
		 */
		if (offset <= curr_off)
Linus Torvalds's avatar
Linus Torvalds committed
1530
			discard_buffer(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1531 1532 1533 1534 1535
		curr_off = next_off;
		bh = next;
	} while (bh != head);

	/*
1536 1537 1538
	 * We release buffers only if the entire page is being invalidated.
	 * The get_block cached value has been unconditionally invalidated,
	 * so real IO is not possible anymore.
Linus Torvalds's avatar
Linus Torvalds committed
1539
	 */
1540 1541 1542 1543
	if (offset == 0)
		ret = try_to_release_page(page, 0);
out:
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1544
}
1545
EXPORT_SYMBOL(block_invalidatepage);
Linus Torvalds's avatar
Linus Torvalds committed
1546

1547 1548
/*
 * We attach and possibly dirty the buffers atomically wrt
1549
 * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1550 1551 1552 1553
 * is already excluded via the page lock.
 */
void create_empty_buffers(struct page *page,
			unsigned long blocksize, unsigned long b_state)
Linus Torvalds's avatar
Linus Torvalds committed
1554 1555 1556 1557 1558 1559
{
	struct buffer_head *bh, *head, *tail;

	head = create_buffers(page, blocksize, 1);
	bh = head;
	do {
1560
		bh->b_state |= b_state;
Linus Torvalds's avatar
Linus Torvalds committed
1561 1562 1563 1564
		tail = bh;
		bh = bh->b_this_page;
	} while (bh);
	tail->b_this_page = head;
1565

1566
	spin_lock(&page->mapping->private_lock);
1567
	if (PageUptodate(page) || PageDirty(page)) {
1568 1569
		bh = head;
		do {
1570 1571 1572 1573
			if (PageDirty(page))
				set_buffer_dirty(bh);
			if (PageUptodate(page))
				set_buffer_uptodate(bh);
1574 1575 1576 1577
			bh = bh->b_this_page;
		} while (bh != head);
	}
	__set_page_buffers(page, head);
1578
	spin_unlock(&page->mapping->private_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1579
}
Linus Torvalds's avatar
Linus Torvalds committed
1580
EXPORT_SYMBOL(create_empty_buffers);
Linus Torvalds's avatar
Linus Torvalds committed
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591

/*
 * We are taking a block for data and we don't want any output from any
 * buffer-cache aliases starting from return from that function and
 * until the moment when something will explicitly mark the buffer
 * dirty (hopefully that will not happen until we will free that block ;-)
 * We don't even need to mark it not-uptodate - nobody can expect
 * anything from a newly allocated buffer anyway. We used to used
 * unmap_buffer() for such invalidation, but that was wrong. We definitely
 * don't want to mark the alias unmapped, for example - it would confuse
 * anyone who might pick it with bread() afterwards...
1592 1593 1594 1595 1596
 *
 * Also..  Note that bforget() doesn't lock the buffer.  So there can
 * be writeout I/O going on against recently-freed buffers.  We don't
 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
 * only if we really need to.  That happens here.
Linus Torvalds's avatar
Linus Torvalds committed
1597
 */
1598
void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
Linus Torvalds's avatar
Linus Torvalds committed
1599 1600 1601
{
	struct buffer_head *old_bh;

1602
	old_bh = __find_get_block(bdev, block, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1603
	if (old_bh) {
1604 1605 1606 1607
#if 0	/* This happens.  Later. */
		if (buffer_dirty(old_bh))
			buffer_error();
#endif
Andrew Morton's avatar
Andrew Morton committed
1608
		clear_buffer_dirty(old_bh);
Linus Torvalds's avatar
Linus Torvalds committed
1609
		wait_on_buffer(old_bh);
Andrew Morton's avatar
Andrew Morton committed
1610
		clear_buffer_req(old_bh);
Linus Torvalds's avatar
Linus Torvalds committed
1611
		__brelse(old_bh);
Linus Torvalds's avatar
Linus Torvalds committed
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
	}
}

/*
 * NOTE! All mapped/uptodate combinations are valid:
 *
 *	Mapped	Uptodate	Meaning
 *
 *	No	No		"unknown" - must do get_block()
 *	No	Yes		"hole" - zero-filled
 *	Yes	No		"allocated" - allocated on disk, not read in
 *	Yes	Yes		"valid" - allocated and up-to-date in memory.
 *
 * "Dirty" is valid only with the last case (mapped+uptodate).
 */

/*
1629 1630 1631 1632
 * While block_write_full_page is writing back the dirty buffers under
 * the page lock, whoever dirtied the buffers may decide to clean them
 * again at any time.  We handle that by only looking at the buffer
 * state inside lock_buffer().
Linus Torvalds's avatar
Linus Torvalds committed
1633
 */
1634 1635
static int __block_write_full_page(struct inode *inode,
			struct page *page, get_block_t *get_block)
Linus Torvalds's avatar
Linus Torvalds committed
1636
{
1637
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
1638
	unsigned long block;
1639
	unsigned long last_block;
Linus Torvalds's avatar
Linus Torvalds committed
1640
	struct buffer_head *bh, *head;
1641
	int nr_underway = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1642

1643
	BUG_ON(!PageLocked(page));
Linus Torvalds's avatar
Linus Torvalds committed
1644

1645
	last_block = (inode->i_size - 1) >> inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
1646

1647 1648 1649
	if (!page_has_buffers(page)) {
		if (S_ISBLK(inode->i_mode))
			buffer_error();
Andrew Morton's avatar
Andrew Morton committed
1650
		if (!PageUptodate(page))
1651 1652 1653 1654
			buffer_error();
		create_empty_buffers(page, 1 << inode->i_blkbits,
					(1 << BH_Dirty)|(1 << BH_Uptodate));
	}
Linus Torvalds's avatar
Linus Torvalds committed
1655

1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
	/*
	 * Be very careful.  We have no exclusion from __set_page_dirty_buffers
	 * here, and the (potentially unmapped) buffers may become dirty at
	 * any time.  If a buffer becomes dirty here after we've inspected it
	 * then we just miss that fact, and the page stays dirty.
	 *
	 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
	 * handle that here by just cleaning them.
	 */

1666 1667
	block = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
	head = page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
1668 1669
	bh = head;

1670 1671 1672 1673
	/*
	 * Get all the dirty buffers mapped to disk addresses and
	 * handle any aliases from the underlying blockdev's mapping.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1674
	do {
1675
		if (block > last_block) {
1676 1677 1678 1679 1680 1681 1682 1683
			/*
			 * mapped buffers outside i_size will occur, because
			 * this page can be outside i_size when there is a
			 * truncate in progress.
			 *
			 * if (buffer_mapped(bh))
			 *	buffer_error();
			 */
1684
			/*
1685
			 * The buffer was zeroed by block_write_full_page()
1686
			 */
1687
			clear_buffer_dirty(bh);
Andrew Morton's avatar
Andrew Morton committed
1688
			set_buffer_uptodate(bh);
1689 1690 1691
		} else if (!buffer_mapped(bh) && buffer_dirty(bh)) {
			if (buffer_new(bh))
				buffer_error();
Linus Torvalds's avatar
Linus Torvalds committed
1692 1693
			err = get_block(inode, block, bh, 1);
			if (err)
1694 1695 1696
				goto recover;
			if (buffer_new(bh)) {
				/* blockdev mappings never come here */
Andrew Morton's avatar
Andrew Morton committed
1697
				clear_buffer_new(bh);
1698 1699
				unmap_underlying_metadata(bh->b_bdev,
							bh->b_blocknr);
1700
			}
Linus Torvalds's avatar
Linus Torvalds committed
1701 1702 1703 1704 1705 1706
		}
		bh = bh->b_this_page;
		block++;
	} while (bh != head);

	do {
1707
		get_bh(bh);
1708
		if (buffer_mapped(bh) && buffer_dirty(bh)) {
1709
			lock_buffer(bh);
1710
			if (test_clear_buffer_dirty(bh)) {
1711 1712
				if (!buffer_uptodate(bh))
					buffer_error();
1713
				mark_buffer_async_write(bh);
1714 1715 1716 1717
			} else {
				unlock_buffer(bh);
			}
		}
Linus Torvalds's avatar
Linus Torvalds committed
1718 1719 1720
		bh = bh->b_this_page;
	} while (bh != head);

1721 1722 1723 1724
	BUG_ON(PageWriteback(page));
	SetPageWriteback(page);		/* Keeps try_to_free_buffers() away */
	unlock_page(page);

1725 1726 1727 1728
	/*
	 * The page may come unlocked any time after the *first* submit_bh()
	 * call.  Be careful with its buffers.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1729
	do {
Linus Torvalds's avatar
Linus Torvalds committed
1730
		struct buffer_head *next = bh->b_this_page;
1731
		if (buffer_async_write(bh)) {
1732 1733 1734 1735
			submit_bh(WRITE, bh);
			nr_underway++;
		}
		put_bh(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1736
		bh = next;
Linus Torvalds's avatar
Linus Torvalds committed
1737 1738
	} while (bh != head);

1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
	err = 0;
done:
	if (nr_underway == 0) {
		/*
		 * The page was marked dirty, but the buffers were
		 * clean.  Someone wrote them back by hand with
		 * ll_rw_block/submit_bh.  A rare case.
		 */
		int uptodate = 1;
		do {
			if (!buffer_uptodate(bh)) {
				uptodate = 0;
				break;
			}
			bh = bh->b_this_page;
		} while (bh != head);
		if (uptodate)
			SetPageUptodate(page);
1757
		end_page_writeback(page);
1758 1759 1760
	}
	return err;
recover:
Andrew Morton's avatar
Andrew Morton committed
1761 1762 1763 1764
	/*
	 * ENOSPC, or some other error.  We may already have added some
	 * blocks to the file, so we need to write these out to avoid
	 * exposing stale data.
1765
	 * The page is currently locked and not marked for writeback
Andrew Morton's avatar
Andrew Morton committed
1766
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1767
	ClearPageUptodate(page);
Andrew Morton's avatar
Andrew Morton committed
1768 1769 1770 1771 1772
	bh = head;
	/* Recovery: lock and submit the mapped buffers */
	do {
		if (buffer_mapped(bh)) {
			lock_buffer(bh);
1773
			mark_buffer_async_write(bh);
1774 1775 1776 1777 1778
		} else {
			/*
			 * The buffer may have been set dirty during
			 * attachment to a dirty page.
			 */
Andrew Morton's avatar
Andrew Morton committed
1779
			clear_buffer_dirty(bh);
Andrew Morton's avatar
Andrew Morton committed
1780 1781 1782 1783 1784
		}
		bh = bh->b_this_page;
	} while (bh != head);
	do {
		struct buffer_head *next = bh->b_this_page;
1785
		if (buffer_async_write(bh)) {
Andrew Morton's avatar
Andrew Morton committed
1786 1787
			set_buffer_uptodate(bh);
			clear_buffer_dirty(bh);
Andrew Morton's avatar
Andrew Morton committed
1788
			submit_bh(WRITE, bh);
1789
			nr_underway++;
Andrew Morton's avatar
Andrew Morton committed
1790 1791 1792
		}
		bh = next;
	} while (bh != head);
1793 1794 1795
	BUG_ON(PageWriteback(page));
	SetPageWriteback(page);
	unlock_page(page);
1796
	goto done;
Linus Torvalds's avatar
Linus Torvalds committed
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
}

static int __block_prepare_write(struct inode *inode, struct page *page,
		unsigned from, unsigned to, get_block_t *get_block)
{
	unsigned block_start, block_end;
	unsigned long block;
	int err = 0;
	unsigned blocksize, bbits;
	struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
	char *kaddr = kmap(page);

1809 1810 1811 1812 1813
	BUG_ON(!PageLocked(page));
	BUG_ON(from > PAGE_CACHE_SIZE);
	BUG_ON(to > PAGE_CACHE_SIZE);
	BUG_ON(from > to);

Linus Torvalds's avatar
Linus Torvalds committed
1814
	blocksize = 1 << inode->i_blkbits;
1815
	if (!page_has_buffers(page))
1816
		create_empty_buffers(page, blocksize, 0);
1817
	head = page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
1818

Linus Torvalds's avatar
Linus Torvalds committed
1819
	bbits = inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
1820 1821 1822 1823
	block = page->index << (PAGE_CACHE_SHIFT - bbits);

	for(bh = head, block_start = 0; bh != head || !block_start;
	    block++, block_start=block_end, bh = bh->b_this_page) {
1824 1825
		block_end = block_start + blocksize;
		if (block_end <= from || block_start >= to) {
1826 1827 1828 1829
			if (PageUptodate(page)) {
				if (!buffer_uptodate(bh))
					set_buffer_uptodate(bh);
			}
Linus Torvalds's avatar
Linus Torvalds committed
1830
			continue;
1831
		}
1832 1833
		if (buffer_new(bh))
			clear_buffer_new(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1834 1835 1836 1837 1838
		if (!buffer_mapped(bh)) {
			err = get_block(inode, block, bh, 1);
			if (err)
				goto out;
			if (buffer_new(bh)) {
Andrew Morton's avatar
Andrew Morton committed
1839
				clear_buffer_new(bh);
1840 1841
				unmap_underlying_metadata(bh->b_bdev,
							bh->b_blocknr);
Andrew Morton's avatar
Andrew Morton committed
1842
				if (PageUptodate(page)) {
1843 1844
					if (!buffer_mapped(bh))
						buffer_error();
Andrew Morton's avatar
Andrew Morton committed
1845
					set_buffer_uptodate(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1846 1847 1848 1849 1850
					continue;
				}
				if (block_end > to)
					memset(kaddr+to, 0, block_end-to);
				if (block_start < from)
1851 1852
					memset(kaddr+block_start,
						0, from-block_start);
Linus Torvalds's avatar
Linus Torvalds committed
1853 1854 1855 1856 1857
				if (block_end > to || block_start < from)
					flush_dcache_page(page);
				continue;
			}
		}
Andrew Morton's avatar
Andrew Morton committed
1858
		if (PageUptodate(page)) {
1859 1860
			if (!buffer_uptodate(bh))
				set_buffer_uptodate(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
			continue; 
		}
		if (!buffer_uptodate(bh) &&
		     (block_start < from || block_end > to)) {
			ll_rw_block(READ, 1, &bh);
			*wait_bh++=bh;
		}
	}
	/*
	 * If we issued read requests - let them complete.
	 */
	while(wait_bh > wait) {
		wait_on_buffer(*--wait_bh);
		if (!buffer_uptodate(*wait_bh))
Andrew Morton's avatar
Andrew Morton committed
1875
			return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
1876 1877 1878
	}
	return 0;
out:
Andrew Morton's avatar
Andrew Morton committed
1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
	/*
	 * Zero out any newly allocated blocks to avoid exposing stale
	 * data.  If BH_New is set, we know that the block was newly
	 * allocated in the above loop.
	 */
	bh = head;
	block_start = 0;
	do {
		block_end = block_start+blocksize;
		if (block_end <= from)
			goto next_bh;
		if (block_start >= to)
			break;
		if (buffer_new(bh)) {
Andrew Morton's avatar
Andrew Morton committed
1893
			clear_buffer_new(bh);
Andrew Morton's avatar
Andrew Morton committed
1894
			if (buffer_uptodate(bh))
1895
				buffer_error();
Andrew Morton's avatar
Andrew Morton committed
1896
			memset(kaddr+block_start, 0, bh->b_size);
Andrew Morton's avatar
Andrew Morton committed
1897
			set_buffer_uptodate(bh);
Andrew Morton's avatar
Andrew Morton committed
1898 1899 1900 1901 1902 1903
			mark_buffer_dirty(bh);
		}
next_bh:
		block_start = block_end;
		bh = bh->b_this_page;
	} while (bh != head);
Linus Torvalds's avatar
Linus Torvalds committed
1904 1905 1906 1907 1908 1909 1910
	return err;
}

static int __block_commit_write(struct inode *inode, struct page *page,
		unsigned from, unsigned to)
{
	unsigned block_start, block_end;
1911
	int partial = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1912 1913 1914
	unsigned blocksize;
	struct buffer_head *bh, *head;

Linus Torvalds's avatar
Linus Torvalds committed
1915
	blocksize = 1 << inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
1916

1917
	for(bh = head = page_buffers(page), block_start = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1918 1919 1920 1921 1922 1923 1924
	    bh != head || !block_start;
	    block_start=block_end, bh = bh->b_this_page) {
		block_end = block_start + blocksize;
		if (block_end <= from || block_start >= to) {
			if (!buffer_uptodate(bh))
				partial = 1;
		} else {
Andrew Morton's avatar
Andrew Morton committed
1925
			set_buffer_uptodate(bh);
1926
			mark_buffer_dirty(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1927 1928 1929 1930
		}
	}

	/*
1931
	 * If this is a partial write which happened to make all buffers
Linus Torvalds's avatar
Linus Torvalds committed
1932
	 * uptodate then we can optimize away a bogus readpage() for
1933
	 * the next read(). Here we 'discover' whether the page went
Linus Torvalds's avatar
Linus Torvalds committed
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
	 * uptodate as a result of this (potentially partial) write.
	 */
	if (!partial)
		SetPageUptodate(page);
	return 0;
}

/*
 * Generic "read page" function for block devices that have the normal
 * get_block functionality. This is most of the block device filesystems.
 * Reads the page asynchronously --- the unlock_buffer() and
Andrew Morton's avatar
Andrew Morton committed
1945
 * set/clear_buffer_uptodate() functions propagate buffer state into the
Linus Torvalds's avatar
Linus Torvalds committed
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
 * page struct once IO has completed.
 */
int block_read_full_page(struct page *page, get_block_t *get_block)
{
	struct inode *inode = page->mapping->host;
	unsigned long iblock, lblock;
	struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
	unsigned int blocksize, blocks;
	int nr, i;

	if (!PageLocked(page))
		PAGE_BUG(page);
Andrew Morton's avatar
Andrew Morton committed
1958
	if (PageUptodate(page))
1959
		buffer_error();
Linus Torvalds's avatar
Linus Torvalds committed
1960
	blocksize = 1 << inode->i_blkbits;
1961
	if (!page_has_buffers(page))
1962
		create_empty_buffers(page, blocksize, 0);
1963
	head = page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
1964

Linus Torvalds's avatar
Linus Torvalds committed
1965 1966 1967
	blocks = PAGE_CACHE_SIZE >> inode->i_blkbits;
	iblock = page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
	lblock = (inode->i_size+blocksize-1) >> inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978
	bh = head;
	nr = 0;
	i = 0;

	do {
		if (buffer_uptodate(bh))
			continue;

		if (!buffer_mapped(bh)) {
			if (iblock < lblock) {
				if (get_block(inode, iblock, bh, 0))
1979
					SetPageError(page);
Linus Torvalds's avatar
Linus Torvalds committed
1980 1981 1982 1983 1984
			}
			if (!buffer_mapped(bh)) {
				memset(kmap(page) + i*blocksize, 0, blocksize);
				flush_dcache_page(page);
				kunmap(page);
Andrew Morton's avatar
Andrew Morton committed
1985
				set_buffer_uptodate(bh);
Linus Torvalds's avatar
Linus Torvalds committed
1986 1987
				continue;
			}
1988 1989 1990 1991
			/*
			 * get_block() might have updated the buffer
			 * synchronously
			 */
Linus Torvalds's avatar
Linus Torvalds committed
1992 1993 1994
			if (buffer_uptodate(bh))
				continue;
		}
1995
		arr[nr++] = bh;
Linus Torvalds's avatar
Linus Torvalds committed
1996 1997 1998 1999
	} while (i++, iblock++, (bh = bh->b_this_page) != head);

	if (!nr) {
		/*
2000 2001
		 * All buffers are uptodate - we can set the page uptodate
		 * as well. But not if get_block() returned an error.
Linus Torvalds's avatar
Linus Torvalds committed
2002
		 */
2003 2004
		if (!PageError(page))
			SetPageUptodate(page);
Andrew Morton's avatar
Andrew Morton committed
2005
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
2006 2007 2008 2009 2010
		return 0;
	}

	/* Stage two: lock the buffers */
	for (i = 0; i < nr; i++) {
2011
		bh = arr[i];
Linus Torvalds's avatar
Linus Torvalds committed
2012
		lock_buffer(bh);
2013
		mark_buffer_async_read(bh);
Linus Torvalds's avatar
Linus Torvalds committed
2014
	}
2015 2016 2017 2018 2019 2020 2021

	/*
	 * Stage 3: start the IO.  Check for uptodateness
	 * inside the buffer lock in case another process reading
	 * the underlying blockdev brought it uptodate (the sct fix).
	 */
	for (i = 0; i < nr; i++) {
2022
		bh = arr[i];
2023
		if (buffer_uptodate(bh))
2024
			end_buffer_async_read(bh, 1);
2025 2026 2027
		else
			submit_bh(READ, bh);
	}
Linus Torvalds's avatar
Linus Torvalds committed
2028 2029 2030
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
/* utility function for filesystems that need to do work on expanding
 * truncates.  Uses prepare/commit_write to allow the filesystem to
 * deal with the hole.  
 */
int generic_cont_expand(struct inode *inode, loff_t size)
{
	struct address_space *mapping = inode->i_mapping;
	struct page *page;
	unsigned long index, offset, limit;
	int err;

	err = -EFBIG;
        limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
	if (limit != RLIM_INFINITY && size > (loff_t)limit) {
		send_sig(SIGXFSZ, current, 0);
		goto out;
	}
	if (size > inode->i_sb->s_maxbytes)
		goto out;

	offset = (size & (PAGE_CACHE_SIZE-1)); /* Within page */

	/* ugh.  in prepare/commit_write, if from==to==start of block, we 
	** skip the prepare.  make sure we never send an offset for the start
	** of a block
	*/
	if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) {
		offset++;
	}
	index = size >> PAGE_CACHE_SHIFT;
	err = -ENOMEM;
	page = grab_cache_page(mapping, index);
	if (!page)
		goto out;
	err = mapping->a_ops->prepare_write(NULL, page, offset, offset);
	if (!err) {
		err = mapping->a_ops->commit_write(NULL, page, offset, offset);
	}
Andrew Morton's avatar
Andrew Morton committed
2069
	unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
2070 2071 2072 2073 2074 2075 2076
	page_cache_release(page);
	if (err > 0)
		err = 0;
out:
	return err;
}

Linus Torvalds's avatar
Linus Torvalds committed
2077 2078 2079 2080 2081
/*
 * For moronic filesystems that do not allow holes in file.
 * We may have to extend the file.
 */

2082 2083
int cont_prepare_write(struct page *page, unsigned offset,
		unsigned to, get_block_t *get_block, unsigned long *bytes)
Linus Torvalds's avatar
Linus Torvalds committed
2084 2085 2086 2087 2088 2089 2090
{
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;
	struct page *new_page;
	unsigned long pgpos;
	long status;
	unsigned zerofrom;
Linus Torvalds's avatar
Linus Torvalds committed
2091
	unsigned blocksize = 1 << inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
2092 2093 2094 2095 2096 2097 2098 2099 2100
	char *kaddr;

	while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) {
		status = -ENOMEM;
		new_page = grab_cache_page(mapping, pgpos);
		if (!new_page)
			goto out;
		/* we might sleep */
		if (*bytes>>PAGE_CACHE_SHIFT != pgpos) {
Andrew Morton's avatar
Andrew Morton committed
2101
			unlock_page(new_page);
Linus Torvalds's avatar
Linus Torvalds committed
2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116
			page_cache_release(new_page);
			continue;
		}
		zerofrom = *bytes & ~PAGE_CACHE_MASK;
		if (zerofrom & (blocksize-1)) {
			*bytes |= (blocksize-1);
			(*bytes)++;
		}
		status = __block_prepare_write(inode, new_page, zerofrom,
						PAGE_CACHE_SIZE, get_block);
		if (status)
			goto out_unmap;
		kaddr = page_address(new_page);
		memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom);
		flush_dcache_page(new_page);
2117 2118
		__block_commit_write(inode, new_page,
				zerofrom, PAGE_CACHE_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
2119
		kunmap(new_page);
Andrew Morton's avatar
Andrew Morton committed
2120
		unlock_page(new_page);
Linus Torvalds's avatar
Linus Torvalds committed
2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
		page_cache_release(new_page);
	}

	if (page->index < pgpos) {
		/* completely inside the area */
		zerofrom = offset;
	} else {
		/* page covers the boundary, find the boundary offset */
		zerofrom = *bytes & ~PAGE_CACHE_MASK;

		/* if we will expand the thing last block will be filled */
		if (to > zerofrom && (zerofrom & (blocksize-1))) {
			*bytes |= (blocksize-1);
			(*bytes)++;
		}

		/* starting below the boundary? Nothing to zero out */
		if (offset <= zerofrom)
			zerofrom = offset;
	}
	status = __block_prepare_write(inode, page, zerofrom, to, get_block);
	if (status)
		goto out1;
	kaddr = page_address(page);
	if (zerofrom < offset) {
		memset(kaddr+zerofrom, 0, offset-zerofrom);
		flush_dcache_page(page);
		__block_commit_write(inode, page, zerofrom, offset);
	}
	return 0;
out1:
	ClearPageUptodate(page);
	kunmap(page);
	return status;

out_unmap:
	ClearPageUptodate(new_page);
	kunmap(new_page);
Andrew Morton's avatar
Andrew Morton committed
2159
	unlock_page(new_page);
Linus Torvalds's avatar
Linus Torvalds committed
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
	page_cache_release(new_page);
out:
	return status;
}

int block_prepare_write(struct page *page, unsigned from, unsigned to,
			get_block_t *get_block)
{
	struct inode *inode = page->mapping->host;
	int err = __block_prepare_write(inode, page, from, to, get_block);
	if (err) {
		ClearPageUptodate(page);
		kunmap(page);
	}
	return err;
}

Linus Torvalds's avatar
Linus Torvalds committed
2177 2178 2179 2180 2181 2182 2183 2184
int block_commit_write(struct page *page, unsigned from, unsigned to)
{
	struct inode *inode = page->mapping->host;
	__block_commit_write(inode,page,from,to);
	kunmap(page);
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
int generic_commit_write(struct file *file, struct page *page,
		unsigned from, unsigned to)
{
	struct inode *inode = page->mapping->host;
	loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
	__block_commit_write(inode,page,from,to);
	kunmap(page);
	if (pos > inode->i_size) {
		inode->i_size = pos;
		mark_inode_dirty(inode);
	}
	return 0;
}

2199 2200
int block_truncate_page(struct address_space *mapping,
			loff_t from, get_block_t *get_block)
Linus Torvalds's avatar
Linus Torvalds committed
2201 2202 2203 2204 2205 2206 2207 2208 2209
{
	unsigned long index = from >> PAGE_CACHE_SHIFT;
	unsigned offset = from & (PAGE_CACHE_SIZE-1);
	unsigned blocksize, iblock, length, pos;
	struct inode *inode = mapping->host;
	struct page *page;
	struct buffer_head *bh;
	int err;

Linus Torvalds's avatar
Linus Torvalds committed
2210
	blocksize = 1 << inode->i_blkbits;
Linus Torvalds's avatar
Linus Torvalds committed
2211 2212 2213 2214 2215 2216 2217
	length = offset & (blocksize - 1);

	/* Block boundary? Nothing to do */
	if (!length)
		return 0;

	length = blocksize - length;
Linus Torvalds's avatar
Linus Torvalds committed
2218
	iblock = index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
Linus Torvalds's avatar
Linus Torvalds committed
2219 2220
	
	page = grab_cache_page(mapping, index);
Linus Torvalds's avatar
Linus Torvalds committed
2221 2222
	err = -ENOMEM;
	if (!page)
Linus Torvalds's avatar
Linus Torvalds committed
2223 2224
		goto out;

2225
	if (!page_has_buffers(page))
2226
		create_empty_buffers(page, blocksize, 0);
Linus Torvalds's avatar
Linus Torvalds committed
2227 2228

	/* Find the buffer that contains "offset" */
2229
	bh = page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
2230 2231 2232 2233 2234 2235 2236 2237 2238
	pos = blocksize;
	while (offset >= pos) {
		bh = bh->b_this_page;
		iblock++;
		pos += blocksize;
	}

	err = 0;
	if (!buffer_mapped(bh)) {
2239 2240
		err = get_block(inode, iblock, bh, 0);
		if (err)
Linus Torvalds's avatar
Linus Torvalds committed
2241
			goto unlock;
2242
		/* unmapped? It's a hole - nothing to do */
Linus Torvalds's avatar
Linus Torvalds committed
2243 2244 2245 2246 2247
		if (!buffer_mapped(bh))
			goto unlock;
	}

	/* Ok, it's mapped. Make sure it's up-to-date */
Andrew Morton's avatar
Andrew Morton committed
2248
	if (PageUptodate(page))
Andrew Morton's avatar
Andrew Morton committed
2249
		set_buffer_uptodate(bh);
Linus Torvalds's avatar
Linus Torvalds committed
2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263

	if (!buffer_uptodate(bh)) {
		err = -EIO;
		ll_rw_block(READ, 1, &bh);
		wait_on_buffer(bh);
		/* Uhhuh. Read error. Complain and punt. */
		if (!buffer_uptodate(bh))
			goto unlock;
	}

	memset(kmap(page) + offset, 0, length);
	flush_dcache_page(page);
	kunmap(page);

2264
	mark_buffer_dirty(bh);
Linus Torvalds's avatar
Linus Torvalds committed
2265 2266 2267
	err = 0;

unlock:
Andrew Morton's avatar
Andrew Morton committed
2268
	unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
2269 2270 2271 2272 2273
	page_cache_release(page);
out:
	return err;
}

2274 2275 2276
/*
 * The generic ->writepage function for buffer-backed address_spaces
 */
Linus Torvalds's avatar
Linus Torvalds committed
2277 2278
int block_write_full_page(struct page *page, get_block_t *get_block)
{
2279 2280
	struct inode * const inode = page->mapping->host;
	const unsigned long end_index = inode->i_size >> PAGE_CACHE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
2281
	unsigned offset;
2282
	char *kaddr;
Linus Torvalds's avatar
Linus Torvalds committed
2283

2284
	/* Is the page fully inside i_size? */
Linus Torvalds's avatar
Linus Torvalds committed
2285 2286 2287
	if (page->index < end_index)
		return __block_write_full_page(inode, page, get_block);

2288
	/* Is the page fully outside i_size? (truncate in progress) */
Linus Torvalds's avatar
Linus Torvalds committed
2289 2290
	offset = inode->i_size & (PAGE_CACHE_SIZE-1);
	if (page->index >= end_index+1 || !offset) {
Andrew Morton's avatar
Andrew Morton committed
2291
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
2292 2293 2294
		return -EIO;
	}

2295 2296 2297 2298 2299 2300
	/* The page straddles i_size */
	kaddr = kmap(page);
	memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset);
	flush_dcache_page(page);
	kunmap(page);
	return __block_write_full_page(inode, page, get_block);
Linus Torvalds's avatar
Linus Torvalds committed
2301 2302
}

Linus Torvalds's avatar
Linus Torvalds committed
2303 2304
sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
			    get_block_t *get_block)
Linus Torvalds's avatar
Linus Torvalds committed
2305 2306 2307 2308 2309 2310 2311 2312 2313
{
	struct buffer_head tmp;
	struct inode *inode = mapping->host;
	tmp.b_state = 0;
	tmp.b_blocknr = 0;
	get_block(inode, block, &tmp, 0);
	return tmp.b_blocknr;
}

2314 2315 2316
int generic_direct_IO(int rw, struct inode *inode,
			struct kiobuf *iobuf, unsigned long blocknr,
			int blocksize, get_block_t *get_block)
Linus Torvalds's avatar
Linus Torvalds committed
2317 2318
{
	int i, nr_blocks, retval;
Linus Torvalds's avatar
Linus Torvalds committed
2319
	sector_t *blocks = iobuf->blocks;
Linus Torvalds's avatar
Linus Torvalds committed
2320 2321 2322 2323

	nr_blocks = iobuf->length / blocksize;
	/* build the blocklist */
	for (i = 0; i < nr_blocks; i++, blocknr++) {
Linus Torvalds's avatar
Linus Torvalds committed
2324 2325
		struct buffer_head bh;

Linus Torvalds's avatar
Linus Torvalds committed
2326 2327 2328
		bh.b_state = 0;
		bh.b_size = blocksize;

Linus Torvalds's avatar
Linus Torvalds committed
2329
		retval = get_block(inode, blocknr, &bh, rw & 1);
Linus Torvalds's avatar
Linus Torvalds committed
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
		if (retval)
			goto out;

		if (rw == READ) {
			if (buffer_new(&bh))
				BUG();
			if (!buffer_mapped(&bh)) {
				/* there was an hole in the filesystem */
				blocks[i] = -1UL;
				continue;
			}
		} else {
			if (buffer_new(&bh))
2343 2344
				unmap_underlying_metadata(bh.b_bdev,
							bh.b_blocknr);
Linus Torvalds's avatar
Linus Torvalds committed
2345 2346 2347 2348 2349 2350
			if (!buffer_mapped(&bh))
				BUG();
		}
		blocks[i] = bh.b_blocknr;
	}

Linus Torvalds's avatar
Linus Torvalds committed
2351
	/* This does not understand multi-device filesystems currently */
2352 2353
	retval = brw_kiovec(rw, 1, &iobuf,
			inode->i_sb->s_bdev, blocks, blocksize);
Linus Torvalds's avatar
Linus Torvalds committed
2354 2355 2356 2357 2358

 out:
	return retval;
}

Linus Torvalds's avatar
Linus Torvalds committed
2359 2360 2361 2362 2363
/*
 * Start I/O on a physical range of kernel memory, defined by a vector
 * of kiobuf structs (much like a user-space iovec list).
 *
 * The kiobuf must already be locked for IO.  IO is submitted
2364
 * asynchronously: you need to check page->locked and page->uptodate.
Linus Torvalds's avatar
Linus Torvalds committed
2365 2366 2367 2368
 *
 * It is up to the caller to make sure that there are enough blocks
 * passed in to completely map the iobufs to disk.
 */
2369 2370
int brw_kiovec(int rw, int nr, struct kiobuf *iovec[],
	       struct block_device *bdev, sector_t b[], int size)
Linus Torvalds's avatar
Linus Torvalds committed
2371 2372 2373
{
	int		transferred;
	int		i;
Linus Torvalds's avatar
Linus Torvalds committed
2374 2375
	int		err;
	struct kiobuf *	iobuf;
Linus Torvalds's avatar
Linus Torvalds committed
2376 2377 2378 2379 2380 2381 2382 2383 2384

	if (!nr)
		return 0;
	
	/* 
	 * First, do some alignment and validity checks 
	 */
	for (i = 0; i < nr; i++) {
		iobuf = iovec[i];
Linus Torvalds's avatar
Linus Torvalds committed
2385
		if ((iobuf->offset & (size-1)) || (iobuf->length & (size-1)))
Linus Torvalds's avatar
Linus Torvalds committed
2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
			return -EINVAL;
		if (!iobuf->nr_pages)
			panic("brw_kiovec: iobuf not initialised");
	}

	/* 
	 * OK to walk down the iovec doing page IO on each page we find. 
	 */
	for (i = 0; i < nr; i++) {
		iobuf = iovec[i];
		iobuf->errno = 0;
Linus Torvalds's avatar
Linus Torvalds committed
2397

2398
		ll_rw_kio(rw, iobuf, bdev, b[i] * (size >> 9));
Linus Torvalds's avatar
Linus Torvalds committed
2399
	}
Linus Torvalds's avatar
Linus Torvalds committed
2400

Linus Torvalds's avatar
Linus Torvalds committed
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412
	/*
	 * now they are all submitted, wait for completion
	 */
	transferred = 0;
	err = 0;
	for (i = 0; i < nr; i++) {
		iobuf = iovec[i];
		kiobuf_wait_for_io(iobuf);
		if (iobuf->errno && !err)
			err = iobuf->errno;
		if (!err)
			transferred += iobuf->length;
Linus Torvalds's avatar
Linus Torvalds committed
2413 2414
	}

Linus Torvalds's avatar
Linus Torvalds committed
2415
	return err ? err : transferred;
Linus Torvalds's avatar
Linus Torvalds committed
2416 2417
}

2418 2419 2420 2421
/*
 * Sanity checks for try_to_free_buffers.
 */
static void check_ttfb_buffer(struct page *page, struct buffer_head *bh)
Linus Torvalds's avatar
Linus Torvalds committed
2422
{
2423
	if (!buffer_uptodate(bh)) {
Andrew Morton's avatar
Andrew Morton committed
2424
		if (PageUptodate(page) && page->mapping
2425 2426 2427 2428 2429 2430
			&& buffer_mapped(bh)	/* discard_buffer */
			&& S_ISBLK(page->mapping->host->i_mode))
		{
			buffer_error();
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
2431 2432
}

Linus Torvalds's avatar
Linus Torvalds committed
2433
/*
2434 2435 2436 2437
 * try_to_free_buffers() checks if all the buffers on this particular page
 * are unused, and releases them if so.
 *
 * Exclusion against try_to_free_buffers may be obtained by either
2438
 * locking the page or by holding its mapping's private_lock.
2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
 *
 * If the page is dirty but all the buffers are clean then we need to
 * be sure to mark the page clean as well.  This is because the page
 * may be against a block device, and a later reattachment of buffers
 * to a dirty page will set *all* buffers dirty.  Which would corrupt
 * filesystem data on the same device.
 *
 * The same applies to regular filesystem pages: if all the buffers are
 * clean then we set the page clean and proceed.  To do that, we require
 * total exclusion from __set_page_dirty_buffers().  That is obtained with
2449
 * private_lock.
2450 2451
 *
 * try_to_free_buffers() is non-blocking.
Linus Torvalds's avatar
Linus Torvalds committed
2452
 */
2453
static inline int buffer_busy(struct buffer_head *bh)
Linus Torvalds's avatar
Linus Torvalds committed
2454
{
2455 2456
	return atomic_read(&bh->b_count) |
		(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
Linus Torvalds's avatar
Linus Torvalds committed
2457 2458
}

2459 2460
static inline int
drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
Linus Torvalds's avatar
Linus Torvalds committed
2461
{
2462
	struct buffer_head *head = page_buffers(page);
2463 2464
	struct buffer_head *bh;
	int was_uptodate = 1;
Linus Torvalds's avatar
Linus Torvalds committed
2465

2466
	bh = head;
Linus Torvalds's avatar
Linus Torvalds committed
2467
	do {
2468 2469 2470 2471 2472
		check_ttfb_buffer(page, bh);
		if (buffer_busy(bh))
			goto failed;
		if (!buffer_uptodate(bh))
			was_uptodate = 0;
Linus Torvalds's avatar
Linus Torvalds committed
2473 2474
		bh = bh->b_this_page;
	} while (bh != head);
Linus Torvalds's avatar
Linus Torvalds committed
2475

Andrew Morton's avatar
Andrew Morton committed
2476
	if (!was_uptodate && PageUptodate(page))
2477
		buffer_error();
Linus Torvalds's avatar
Linus Torvalds committed
2478

Linus Torvalds's avatar
Linus Torvalds committed
2479
	do {
2480
		struct buffer_head *next = bh->b_this_page;
Linus Torvalds's avatar
Linus Torvalds committed
2481

2482 2483
		if (!list_empty(&bh->b_assoc_buffers))
			__remove_assoc_queue(bh);
2484 2485
		bh = next;
	} while (bh != head);
2486
	*buffers_to_free = head;
2487
	__clear_page_buffers(page);
Linus Torvalds's avatar
Linus Torvalds committed
2488
	return 1;
2489 2490
failed:
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
2491 2492
}

2493
int try_to_free_buffers(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
2494
{
2495
	struct address_space * const mapping = page->mapping;
2496
	struct buffer_head *buffers_to_free = NULL;
2497
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
2498

Linus Torvalds's avatar
Linus Torvalds committed
2499
	BUG_ON(!PageLocked(page));
2500 2501
	if (PageWriteback(page))
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
2502

2503 2504 2505 2506
	if (mapping == NULL) {		/* swapped-in anon page */
		ret = drop_buffers(page, &buffers_to_free);
		goto out;
	}
Linus Torvalds's avatar
Linus Torvalds committed
2507

2508
	spin_lock(&mapping->private_lock);
2509
	ret = drop_buffers(page, &buffers_to_free);
2510 2511 2512 2513 2514 2515 2516 2517 2518
	if (ret && !PageSwapCache(page)) {
		/*
		 * If the filesystem writes its buffers by hand (eg ext3)
		 * then we can have clean buffers against a dirty page.  We
		 * clean the page here; otherwise later reattachment of buffers
		 * could encounter a non-uptodate page, which is unresolvable.
		 * This only applies in the rare case where try_to_free_buffers
		 * succeeds but the page is not freed.
		 */
2519
		ClearPageDirty(page);
2520
	}
2521
	spin_unlock(&mapping->private_lock);
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531
out:
	if (buffers_to_free) {
		struct buffer_head *bh = buffers_to_free;

		do {
			struct buffer_head *next = bh->b_this_page;
			free_buffer_head(bh);
			bh = next;
		} while (bh != buffers_to_free);
	}
2532
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
2533
}
Linus Torvalds's avatar
Linus Torvalds committed
2534
EXPORT_SYMBOL(try_to_free_buffers);
Linus Torvalds's avatar
Linus Torvalds committed
2535 2536 2537

int block_sync_page(struct page *page)
{
Jens Axboe's avatar
Jens Axboe committed
2538
	blk_run_queues();
Linus Torvalds's avatar
Linus Torvalds committed
2539 2540 2541
	return 0;
}

2542 2543 2544 2545
/*
 * There are no bdflush tunables left.  But distributions are
 * still running obsolete flush daemons, so we terminate them here.
 */
Linus Torvalds's avatar
Linus Torvalds committed
2546 2547 2548 2549
asmlinkage long sys_bdflush(int func, long data)
{
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
2550
	if (func == 1)
Linus Torvalds's avatar
Linus Torvalds committed
2551 2552 2553 2554
		do_exit(0);
	return 0;
}

2555 2556 2557 2558 2559 2560
/*
 * Buffer-head allocation
 */
static kmem_cache_t *bh_cachep;
static mempool_t *bh_mempool;

2561
struct buffer_head *alloc_buffer_head(void)
2562 2563 2564 2565 2566 2567 2568
{
	return mempool_alloc(bh_mempool, GFP_NOFS);
}
EXPORT_SYMBOL(alloc_buffer_head);

void free_buffer_head(struct buffer_head *bh)
{
2569
	BUG_ON(!list_empty(&bh->b_assoc_buffers));
2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
	mempool_free(bh, bh_mempool);
}
EXPORT_SYMBOL(free_buffer_head);

static void init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long flags)
{
	if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
			    SLAB_CTOR_CONSTRUCTOR) {
		struct buffer_head * bh = (struct buffer_head *)data;

		memset(bh, 0, sizeof(*bh));
2581
		INIT_LIST_HEAD(&bh->b_assoc_buffers);
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599
	}
}

static void *bh_mempool_alloc(int gfp_mask, void *pool_data)
{
	return kmem_cache_alloc(bh_cachep, gfp_mask);
}

static void bh_mempool_free(void *element, void *pool_data)
{
	return kmem_cache_free(bh_cachep, element);
}

#define NR_RESERVED (10*MAX_BUF_PER_PAGE)
#define MAX_UNUSED_BUFFERS NR_RESERVED+20

void __init buffer_init(void)
{
Andrew Morton's avatar
Andrew Morton committed
2600 2601
	int i;

2602 2603 2604
	for (i = 0; i < NR_CPUS; i++)
		spin_lock_init(&bh_lrus[i].lock);

2605 2606 2607 2608 2609
	bh_cachep = kmem_cache_create("buffer_head",
			sizeof(struct buffer_head), 0,
			SLAB_HWCACHE_ALIGN, init_buffer_head, NULL);
	bh_mempool = mempool_create(MAX_UNUSED_BUFFERS, bh_mempool_alloc,
				bh_mempool_free, NULL);
Andrew Morton's avatar
Andrew Morton committed
2610 2611
	for (i = 0; i < ARRAY_SIZE(bh_wait_queue_heads); i++)
		init_waitqueue_head(&bh_wait_queue_heads[i].wqh);
2612
}