[PATCH] AIO support for raw/O_DIRECT

Patch from Badari Pulavarty <pbadari@us.ibm.com> and myself

This patch adds the infrastructure for performing asynchronous (AIO) blockdev
direct-IO.

- Adds generic_file_aio_write_nolock() and make other
  generic_file_*_write() to use it.

- Modify generic_file_direct_IO() and ->direct_IO() functions to take
  "kiocb *" instead of "file *".

- Renames generic_direct_IO() to blockdev_direct_IO().

- Move generic_file_direct_IO() to mm/filemap.c (it is not
  blockdev-specific, whereas the rest of fs/direct-io.c is).

- Add AIO read/write support to the raw driver.

drivers/char/raw.c

@@ -212,9 +212,20 @@ static ssize_t raw_file_write(struct file *file, const char *buf,
 	return generic_file_write_nolock(file, &local_iov, 1, ppos);
 }
 
+static ssize_t raw_file_aio_write(struct kiocb *iocb, const char *buf,
+					size_t count, loff_t pos)
+{
+	struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };
+
+	return generic_file_aio_write_nolock(iocb, &local_iov, 1, &iocb->ki_pos);
+}
+
+
 static struct file_operations raw_fops = {
 	.read	=	generic_file_read,
+	.aio_read = 	generic_file_aio_read,
 	.write	=	raw_file_write,
+	.aio_write = 	raw_file_aio_write,
 	.open	=	raw_open,
 	.release=	raw_release,
 	.ioctl	=	raw_ioctl,

fs/block_dev.c

@@ -118,12 +118,13 @@ blkdev_get_blocks(struct inode *inode, sector_t iblock,
 }
 
 static int
-blkdev_direct_IO(int rw, struct file *file, const struct iovec *iov,
+blkdev_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
 			loff_t offset, unsigned long nr_segs)
 {
+	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-	return generic_direct_IO(rw, inode, inode->i_bdev, iov, offset,
+	return blockdev_direct_IO(rw, iocb, inode, inode->i_bdev, iov, offset,
 				nr_segs, blkdev_get_blocks);
 }
 

fs/direct-io.c

@@ -845,9 +845,9 @@ direct_io_worker(int rw, struct inode *inode, const struct iovec *iov,
  * This is a library function for use by filesystem drivers.
  */
 int
-generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev, 
-	const struct iovec *iov, loff_t offset, unsigned long nr_segs, 
-	get_blocks_t get_blocks)
+blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
+	struct block_device *bdev, const struct iovec *iov, loff_t offset, 
+	unsigned long nr_segs, get_blocks_t get_blocks)
 {
 	int seg;
 	size_t size;
@@ -886,25 +886,3 @@ generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev,
 out:
 	return retval;
 }
-
-ssize_t
-generic_file_direct_IO(int rw, struct file *file, const struct iovec *iov,
-	loff_t offset, unsigned long nr_segs)
-{
-	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
-	ssize_t retval;
-
-	if (mapping->nrpages) {
-		retval = filemap_fdatawrite(mapping);
-		if (retval == 0)
-			retval = filemap_fdatawait(mapping);
-		if (retval)
-			goto out;
-	}
-
-	retval = mapping->a_ops->direct_IO(rw, file, iov, offset, nr_segs);
-	if (rw == WRITE && mapping->nrpages)
-		invalidate_inode_pages2(mapping);
-out:
-	return retval;
-}

fs/ext2/inode.c

@@ -652,12 +652,13 @@ ext2_get_blocks(struct inode *inode, sector_t iblock, unsigned long max_blocks,
 }
 
 static int
-ext2_direct_IO(int rw, struct file *file, const struct iovec *iov,
+ext2_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
 			loff_t offset, unsigned long nr_segs)
 {
+	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-	return generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov,
+	return blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov,
 				offset, nr_segs, ext2_get_blocks);
 }
 

fs/ext3/inode.c

@@ -1405,10 +1405,11 @@ static int ext3_releasepage(struct page *page, int wait)
  * If the O_DIRECT write is intantiating holes inside i_size and the machine
  * crashes then stale disk data _may_ be exposed inside the file.
  */
-static int ext3_direct_IO(int rw, struct file *file,
+static int ext3_direct_IO(int rw, struct kiocb *iocb,
 			const struct iovec *iov, loff_t offset,
 			unsigned long nr_segs)
 {
+	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 	struct ext3_inode_info *ei = EXT3_I(inode);
 	handle_t *handle = NULL;
@@ -1437,8 +1438,8 @@ static int ext3_direct_IO(int rw, struct file *file,
 		}
 	}
 
-	ret = generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov, offset,
-				nr_segs, ext3_direct_io_get_blocks);
+	ret = blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov, 
+				offset, nr_segs, ext3_direct_io_get_blocks);
 
 out_stop:
 	if (handle) {

fs/jfs/inode.c

@@ -310,12 +310,13 @@ static sector_t jfs_bmap(struct address_space *mapping, sector_t block)
 	return generic_block_bmap(mapping, block, jfs_get_block);
 }
 
-static int jfs_direct_IO(int rw, struct file *file, const struct iovec *iov,
+static int jfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
 			loff_t offset, unsigned long nr_segs)
 {
+	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-	return generic_direct_IO(rw, inode, inode->i_sb->s_bdev, iov,
+	return blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov,
 				offset, nr_segs, jfs_get_blocks);
 }
 

fs/nfs/direct.c

@@ -235,7 +235,7 @@ do_nfs_direct_IO(int rw, const struct inode *inode,
  * this function to do a write.
  */
 int
-nfs_direct_IO(int rw, struct file *file, const struct iovec *iov,
+nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
 		loff_t offset, unsigned long nr_segs)
 {
 	/* None of this works yet, so prevent it from compiling. */

fs/xfs/linux/xfs_aops.c

@@ -625,14 +625,15 @@ linvfs_get_blocks_direct(
 STATIC int
 linvfs_direct_IO(
 	int			rw,
-	struct file		*file,
+	struct kiocb		*iocb,
 	const struct iovec	*iov,
 	loff_t			offset,
 	unsigned long		nr_segs)
 {
+	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
 
-        return generic_direct_IO(rw, inode, NULL,
+        return blockdev_direct_IO(rw, iocb, inode, NULL,
 			iov, offset, nr_segs, linvfs_get_blocks_direct);
 }
 

include/linux/fs.h

@@ -276,6 +276,7 @@ struct iattr {
 struct page;
 struct address_space;
 struct writeback_control;
+struct kiocb;
 
 struct address_space_operations {
 	int (*writepage)(struct page *page, struct writeback_control *wbc);
@@ -301,7 +302,7 @@ struct address_space_operations {
 	sector_t (*bmap)(struct address_space *, sector_t);
 	int (*invalidatepage) (struct page *, unsigned long);
 	int (*releasepage) (struct page *, int);
-	int (*direct_IO)(int, struct file *, const struct iovec *iov,
+	int (*direct_IO)(int, struct kiocb *, const struct iovec *iov,
 			loff_t offset, unsigned long nr_segs);
 };
 
@@ -736,7 +737,6 @@ typedef int (*read_actor_t)(read_descriptor_t *, struct page *, unsigned long, u
  * read, write, poll, fsync, readv, writev can be called
  *   without the big kernel lock held in all filesystems.
  */
-struct kiocb;
 struct file_operations {
 	struct module *owner;
 	loff_t (*llseek) (struct file *, loff_t, int);
@@ -1234,6 +1234,8 @@ extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *);
 extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *);
 extern ssize_t generic_file_aio_read(struct kiocb *, char *, size_t, loff_t);
 extern ssize_t generic_file_aio_write(struct kiocb *, const char *, size_t, loff_t);
+extern ssize_t generic_file_aio_write_nolock(struct kiocb *, const struct iovec *,
+				unsigned long, loff_t *);
 extern ssize_t do_sync_read(struct file *filp, char *buf, size_t len, loff_t *ppos);
 extern ssize_t do_sync_write(struct file *filp, const char *buf, size_t len, loff_t *ppos);
 ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov,
@@ -1243,10 +1245,11 @@ extern void do_generic_mapping_read(struct address_space *, struct file_ra_state
 				    loff_t *, read_descriptor_t *, read_actor_t);
 extern void
 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
-extern ssize_t generic_file_direct_IO(int rw, struct file *file,
+extern ssize_t generic_file_direct_IO(int rw, struct kiocb *iocb,
 	const struct iovec *iov, loff_t offset, unsigned long nr_segs);
-extern int generic_direct_IO(int rw, struct inode *inode, struct block_device *bdev,
-	const struct iovec *iov, loff_t offset, unsigned long nr_segs, get_blocks_t *get_blocks);
+extern int blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, 
+	struct block_device *bdev, const struct iovec *iov, loff_t offset, 
+	unsigned long nr_segs, get_blocks_t *get_blocks);
 extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 
 	unsigned long nr_segs, loff_t *ppos);
 ssize_t generic_file_writev(struct file *filp, const struct iovec *iov, 

kernel/ksyms.c

@@ -215,7 +215,7 @@ EXPORT_SYMBOL(ll_rw_block);
 EXPORT_SYMBOL(submit_bh);
 EXPORT_SYMBOL(unlock_buffer);
 EXPORT_SYMBOL(__wait_on_buffer);
-EXPORT_SYMBOL(generic_direct_IO);
+EXPORT_SYMBOL(blockdev_direct_IO);
 EXPORT_SYMBOL(block_write_full_page);
 EXPORT_SYMBOL(block_read_full_page);
 EXPORT_SYMBOL(block_prepare_write);

mm/filemap.c

@@ -805,7 +805,7 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 				nr_segs = iov_shorten((struct iovec *)iov,
 							nr_segs, count);
 			}
-			retval = generic_file_direct_IO(READ, filp,
+			retval = generic_file_direct_IO(READ, iocb,
 					iov, pos, nr_segs);
 			if (retval > 0)
 				*ppos = pos + retval;
@@ -1532,9 +1532,10 @@ filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
  *							okir@monad.swb.de
  */
 ssize_t
-generic_file_write_nolock(struct file *file, const struct iovec *iov,
+generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
 				unsigned long nr_segs, loff_t *ppos)
 {
+	struct file *file = iocb->ki_filp;
 	struct address_space * mapping = file->f_dentry->d_inode->i_mapping;
 	struct address_space_operations *a_ops = mapping->a_ops;
 	size_t ocount;		/* original count */
@@ -1674,7 +1675,7 @@ generic_file_write_nolock(struct file *file, const struct iovec *iov,
 		if (count != ocount)
 			nr_segs = iov_shorten((struct iovec *)iov,
 						nr_segs, count);
-		written = generic_file_direct_IO(WRITE, file,
+		written = generic_file_direct_IO(WRITE, iocb,
 					iov, pos, nr_segs);
 		if (written > 0) {
 			loff_t end = pos + written;
@@ -1786,12 +1787,39 @@ generic_file_write_nolock(struct file *file, const struct iovec *iov,
 	return err;
 }
 
+ssize_t
+generic_file_write_nolock(struct file *file, const struct iovec *iov,
+				unsigned long nr_segs, loff_t *ppos)
+{
+	struct kiocb kiocb;
+	ssize_t ret;
+
+	init_sync_kiocb(&kiocb, file);
+	ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
+	if (-EIOCBQUEUED == ret)
+		ret = wait_on_sync_kiocb(&kiocb);
+	return ret;
+}
+
 ssize_t generic_file_aio_write(struct kiocb *iocb, const char *buf,
 			       size_t count, loff_t pos)
 {
-	return generic_file_write(iocb->ki_filp, buf, count, &iocb->ki_pos);
+	struct file *file = iocb->ki_filp;
+	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
+	int err;
+	struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };
+
+	BUG_ON(iocb->ki_pos != pos);
+
+	down(&inode->i_sem);
+	err = generic_file_aio_write_nolock(iocb, &local_iov, 1, 
+						&iocb->ki_pos);
+	up(&inode->i_sem);
+
+	return err;
 }
 EXPORT_SYMBOL(generic_file_aio_write);
+EXPORT_SYMBOL(generic_file_aio_write_nolock);
 
 ssize_t generic_file_write(struct file *file, const char *buf,
 			   size_t count, loff_t *ppos)
@@ -1831,3 +1859,26 @@ ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
 	up(&inode->i_sem);
 	return ret;
 }
+
+ssize_t
+generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
+	loff_t offset, unsigned long nr_segs)
+{
+	struct file *file = iocb->ki_filp;
+	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
+	ssize_t retval;
+
+	if (mapping->nrpages) {
+		retval = filemap_fdatawrite(mapping);
+		if (retval == 0)
+			retval = filemap_fdatawait(mapping);
+		if (retval)
+			goto out;
+	}
+
+	retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
+	if (rw == WRITE && mapping->nrpages)
+		invalidate_inode_pages2(mapping);
+out:
+	return retval;
+}