Commit 17335dcc authored by Dmitry Monakhov's avatar Dmitry Monakhov Committed by Theodore Ts'o

ext4: serialize dio nonlocked reads with defrag workers

Inode's block defrag and ext4_change_inode_journal_flag() may
affect nonlocked DIO reads result, so proper synchronization
required.

- Add missed inode_dio_wait() calls where appropriate
- Check inode state under extra i_dio_count reference.
Reviewed-by: default avatarJan Kara <jack@suse.cz>
Signed-off-by: default avatarDmitry Monakhov <dmonakhov@openvz.org>
Signed-off-by: default avatar"Theodore Ts'o" <tytso@mit.edu>
parent 28a535f9
...@@ -1358,6 +1358,8 @@ enum { ...@@ -1358,6 +1358,8 @@ enum {
EXT4_STATE_DIO_UNWRITTEN, /* need convert on dio done*/ EXT4_STATE_DIO_UNWRITTEN, /* need convert on dio done*/
EXT4_STATE_NEWENTRY, /* File just added to dir */ EXT4_STATE_NEWENTRY, /* File just added to dir */
EXT4_STATE_DELALLOC_RESERVED, /* blks already reserved for delalloc */ EXT4_STATE_DELALLOC_RESERVED, /* blks already reserved for delalloc */
EXT4_STATE_DIOREAD_LOCK, /* Disable support for dio read
nolocking */
}; };
#define EXT4_INODE_BIT_FNS(name, field, offset) \ #define EXT4_INODE_BIT_FNS(name, field, offset) \
...@@ -2469,6 +2471,21 @@ static inline void set_bitmap_uptodate(struct buffer_head *bh) ...@@ -2469,6 +2471,21 @@ static inline void set_bitmap_uptodate(struct buffer_head *bh)
set_bit(BH_BITMAP_UPTODATE, &(bh)->b_state); set_bit(BH_BITMAP_UPTODATE, &(bh)->b_state);
} }
/*
* Disable DIO read nolock optimization, so new dioreaders will be forced
* to grab i_mutex
*/
static inline void ext4_inode_block_unlocked_dio(struct inode *inode)
{
ext4_set_inode_state(inode, EXT4_STATE_DIOREAD_LOCK);
smp_mb();
}
static inline void ext4_inode_resume_unlocked_dio(struct inode *inode)
{
smp_mb();
ext4_clear_inode_state(inode, EXT4_STATE_DIOREAD_LOCK);
}
#define in_range(b, first, len) ((b) >= (first) && (b) <= (first) + (len) - 1) #define in_range(b, first, len) ((b) >= (first) && (b) <= (first) + (len) - 1)
/* For ioend & aio unwritten conversion wait queues */ /* For ioend & aio unwritten conversion wait queues */
......
...@@ -810,11 +810,25 @@ ssize_t ext4_ind_direct_IO(int rw, struct kiocb *iocb, ...@@ -810,11 +810,25 @@ ssize_t ext4_ind_direct_IO(int rw, struct kiocb *iocb,
if (unlikely(!list_empty(&ei->i_completed_io_list))) if (unlikely(!list_empty(&ei->i_completed_io_list)))
ext4_flush_completed_IO(inode); ext4_flush_completed_IO(inode);
/*
* Nolock dioread optimization may be dynamically disabled
* via ext4_inode_block_unlocked_dio(). Check inode's state
* while holding extra i_dio_count ref.
*/
atomic_inc(&inode->i_dio_count);
smp_mb();
if (unlikely(ext4_test_inode_state(inode,
EXT4_STATE_DIOREAD_LOCK))) {
inode_dio_done(inode);
goto locked;
}
ret = __blockdev_direct_IO(rw, iocb, inode, ret = __blockdev_direct_IO(rw, iocb, inode,
inode->i_sb->s_bdev, iov, inode->i_sb->s_bdev, iov,
offset, nr_segs, offset, nr_segs,
ext4_get_block, NULL, NULL, 0); ext4_get_block, NULL, NULL, 0);
inode_dio_done(inode);
} else { } else {
locked:
ret = blockdev_direct_IO(rw, iocb, inode, iov, ret = blockdev_direct_IO(rw, iocb, inode, iov,
offset, nr_segs, ext4_get_block); offset, nr_segs, ext4_get_block);
......
...@@ -4720,6 +4720,10 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val) ...@@ -4720,6 +4720,10 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val)
return err; return err;
} }
/* Wait for all existing dio workers */
ext4_inode_block_unlocked_dio(inode);
inode_dio_wait(inode);
jbd2_journal_lock_updates(journal); jbd2_journal_lock_updates(journal);
/* /*
...@@ -4739,6 +4743,7 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val) ...@@ -4739,6 +4743,7 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val)
ext4_set_aops(inode); ext4_set_aops(inode);
jbd2_journal_unlock_updates(journal); jbd2_journal_unlock_updates(journal);
ext4_inode_resume_unlocked_dio(inode);
/* Finally we can mark the inode as dirty. */ /* Finally we can mark the inode as dirty. */
......
...@@ -1323,6 +1323,12 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, ...@@ -1323,6 +1323,12 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp,
/* Protect orig and donor inodes against a truncate */ /* Protect orig and donor inodes against a truncate */
mext_inode_double_lock(orig_inode, donor_inode); mext_inode_double_lock(orig_inode, donor_inode);
/* Wait for all existing dio workers */
ext4_inode_block_unlocked_dio(orig_inode);
ext4_inode_block_unlocked_dio(donor_inode);
inode_dio_wait(orig_inode);
inode_dio_wait(donor_inode);
/* Protect extent tree against block allocations via delalloc */ /* Protect extent tree against block allocations via delalloc */
double_down_write_data_sem(orig_inode, donor_inode); double_down_write_data_sem(orig_inode, donor_inode);
/* Check the filesystem environment whether move_extent can be done */ /* Check the filesystem environment whether move_extent can be done */
...@@ -1521,6 +1527,8 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, ...@@ -1521,6 +1527,8 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp,
kfree(holecheck_path); kfree(holecheck_path);
} }
double_up_write_data_sem(orig_inode, donor_inode); double_up_write_data_sem(orig_inode, donor_inode);
ext4_inode_resume_unlocked_dio(orig_inode);
ext4_inode_resume_unlocked_dio(donor_inode);
mext_inode_double_unlock(orig_inode, donor_inode); mext_inode_double_unlock(orig_inode, donor_inode);
return ret; return ret;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment