Commit fb830fc5 authored by Chao Yu's avatar Chao Yu Committed by Jaegeuk Kim

f2fs: introduce io_list for serialize data/node IOs

Serialize data/node IOs by using fifo list instead of mutex lock,
it will help to enhance concurrency of f2fs, meanwhile keeping LFS
IO semantics.
Signed-off-by: default avatarChao Yu <yuchao0@huawei.com>
Signed-off-by: default avatarJaegeuk Kim <jaegeuk@kernel.org>
parent e41e6d75
...@@ -162,6 +162,7 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, ...@@ -162,6 +162,7 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
.op = REQ_OP_READ, .op = REQ_OP_READ,
.op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD, .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
.encrypted_page = NULL, .encrypted_page = NULL,
.in_list = false,
}; };
struct blk_plug plug; struct blk_plug plug;
......
...@@ -397,6 +397,20 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio) ...@@ -397,6 +397,20 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio)
f2fs_bug_on(sbi, is_read_io(fio->op)); f2fs_bug_on(sbi, is_read_io(fio->op));
down_write(&io->io_rwsem);
next:
if (fio->in_list) {
spin_lock(&io->io_lock);
if (list_empty(&io->io_list)) {
spin_unlock(&io->io_lock);
goto out_fail;
}
fio = list_first_entry(&io->io_list,
struct f2fs_io_info, list);
list_del(&fio->list);
spin_unlock(&io->io_lock);
}
if (fio->old_blkaddr != NEW_ADDR) if (fio->old_blkaddr != NEW_ADDR)
verify_block_addr(sbi, fio->old_blkaddr); verify_block_addr(sbi, fio->old_blkaddr);
verify_block_addr(sbi, fio->new_blkaddr); verify_block_addr(sbi, fio->new_blkaddr);
...@@ -408,8 +422,6 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio) ...@@ -408,8 +422,6 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio)
inc_page_count(sbi, WB_DATA_TYPE(bio_page)); inc_page_count(sbi, WB_DATA_TYPE(bio_page));
down_write(&io->io_rwsem);
if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 || if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
(io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) || (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
!__same_bdev(sbi, fio->new_blkaddr, io->bio))) !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
...@@ -434,9 +446,13 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio) ...@@ -434,9 +446,13 @@ int f2fs_submit_page_write(struct f2fs_io_info *fio)
io->last_block_in_bio = fio->new_blkaddr; io->last_block_in_bio = fio->new_blkaddr;
f2fs_trace_ios(fio, 0); f2fs_trace_ios(fio, 0);
trace_f2fs_submit_page_write(fio->page, fio);
if (fio->in_list)
goto next;
out_fail: out_fail:
up_write(&io->io_rwsem); up_write(&io->io_rwsem);
trace_f2fs_submit_page_write(fio->page, fio);
return err; return err;
} }
...@@ -749,7 +765,7 @@ static int __allocate_data_block(struct dnode_of_data *dn) ...@@ -749,7 +765,7 @@ static int __allocate_data_block(struct dnode_of_data *dn)
set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
&sum, CURSEG_WARM_DATA); &sum, CURSEG_WARM_DATA, NULL, false);
set_data_blkaddr(dn); set_data_blkaddr(dn);
/* update i_size */ /* update i_size */
......
...@@ -815,8 +815,10 @@ struct f2fs_io_info { ...@@ -815,8 +815,10 @@ struct f2fs_io_info {
block_t old_blkaddr; /* old block address before Cow */ block_t old_blkaddr; /* old block address before Cow */
struct page *page; /* page to be written */ struct page *page; /* page to be written */
struct page *encrypted_page; /* encrypted page */ struct page *encrypted_page; /* encrypted page */
struct list_head list; /* serialize IOs */
bool submitted; /* indicate IO submission */ bool submitted; /* indicate IO submission */
int need_lock; /* indicate we need to lock cp_rwsem */ int need_lock; /* indicate we need to lock cp_rwsem */
bool in_list; /* indicate fio is in io_list */
}; };
#define is_read_io(rw) ((rw) == READ) #define is_read_io(rw) ((rw) == READ)
...@@ -826,6 +828,8 @@ struct f2fs_bio_info { ...@@ -826,6 +828,8 @@ struct f2fs_bio_info {
sector_t last_block_in_bio; /* last block number */ sector_t last_block_in_bio; /* last block number */
struct f2fs_io_info fio; /* store buffered io info. */ struct f2fs_io_info fio; /* store buffered io info. */
struct rw_semaphore io_rwsem; /* blocking op for bio */ struct rw_semaphore io_rwsem; /* blocking op for bio */
spinlock_t io_lock; /* serialize DATA/NODE IOs */
struct list_head io_list; /* track fios */
}; };
#define FDEV(i) (sbi->devs[i]) #define FDEV(i) (sbi->devs[i])
...@@ -2294,7 +2298,8 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, ...@@ -2294,7 +2298,8 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
bool recover_newaddr); bool recover_newaddr);
void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
block_t old_blkaddr, block_t *new_blkaddr, block_t old_blkaddr, block_t *new_blkaddr,
struct f2fs_summary *sum, int type); struct f2fs_summary *sum, int type,
struct f2fs_io_info *fio, bool add_list);
void f2fs_wait_on_page_writeback(struct page *page, void f2fs_wait_on_page_writeback(struct page *page,
enum page_type type, bool ordered); enum page_type type, bool ordered);
void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi, void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
......
...@@ -600,6 +600,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx, ...@@ -600,6 +600,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx,
.op = REQ_OP_READ, .op = REQ_OP_READ,
.op_flags = 0, .op_flags = 0,
.encrypted_page = NULL, .encrypted_page = NULL,
.in_list = false,
}; };
struct dnode_of_data dn; struct dnode_of_data dn;
struct f2fs_summary sum; struct f2fs_summary sum;
...@@ -643,7 +644,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx, ...@@ -643,7 +644,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx,
fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr; fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr, allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
&sum, CURSEG_COLD_DATA); &sum, CURSEG_COLD_DATA, NULL, false);
fio.encrypted_page = pagecache_get_page(META_MAPPING(fio.sbi), newaddr, fio.encrypted_page = pagecache_get_page(META_MAPPING(fio.sbi), newaddr,
FGP_LOCK | FGP_CREAT, GFP_NOFS); FGP_LOCK | FGP_CREAT, GFP_NOFS);
......
...@@ -2109,7 +2109,8 @@ static int __get_segment_type(struct f2fs_io_info *fio) ...@@ -2109,7 +2109,8 @@ static int __get_segment_type(struct f2fs_io_info *fio)
void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
block_t old_blkaddr, block_t *new_blkaddr, block_t old_blkaddr, block_t *new_blkaddr,
struct f2fs_summary *sum, int type) struct f2fs_summary *sum, int type,
struct f2fs_io_info *fio, bool add_list)
{ {
struct sit_info *sit_i = SIT_I(sbi); struct sit_info *sit_i = SIT_I(sbi);
struct curseg_info *curseg = CURSEG_I(sbi, type); struct curseg_info *curseg = CURSEG_I(sbi, type);
...@@ -2145,6 +2146,17 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, ...@@ -2145,6 +2146,17 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
if (page && IS_NODESEG(type)) if (page && IS_NODESEG(type))
fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
if (add_list) {
struct f2fs_bio_info *io;
INIT_LIST_HEAD(&fio->list);
fio->in_list = true;
io = sbi->write_io[fio->type] + fio->temp;
spin_lock(&io->io_lock);
list_add_tail(&fio->list, &io->io_list);
spin_unlock(&io->io_lock);
}
mutex_unlock(&curseg->curseg_mutex); mutex_unlock(&curseg->curseg_mutex);
} }
...@@ -2153,11 +2165,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) ...@@ -2153,11 +2165,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
int type = __get_segment_type(fio); int type = __get_segment_type(fio);
int err; int err;
if (fio->type == NODE || fio->type == DATA)
mutex_lock(&fio->sbi->wio_mutex[fio->type][fio->temp]);
reallocate: reallocate:
allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr, allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
&fio->new_blkaddr, sum, type); &fio->new_blkaddr, sum, type, fio, true);
/* writeout dirty page into bdev */ /* writeout dirty page into bdev */
err = f2fs_submit_page_write(fio); err = f2fs_submit_page_write(fio);
...@@ -2165,9 +2175,6 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) ...@@ -2165,9 +2175,6 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
fio->old_blkaddr = fio->new_blkaddr; fio->old_blkaddr = fio->new_blkaddr;
goto reallocate; goto reallocate;
} }
if (fio->type == NODE || fio->type == DATA)
mutex_unlock(&fio->sbi->wio_mutex[fio->type][fio->temp]);
} }
void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
...@@ -2181,6 +2188,7 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) ...@@ -2181,6 +2188,7 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
.new_blkaddr = page->index, .new_blkaddr = page->index,
.page = page, .page = page,
.encrypted_page = NULL, .encrypted_page = NULL,
.in_list = false,
}; };
if (unlikely(page->index >= MAIN_BLKADDR(sbi))) if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
......
...@@ -1983,6 +1983,8 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent) ...@@ -1983,6 +1983,8 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
init_rwsem(&sbi->write_io[i][j].io_rwsem); init_rwsem(&sbi->write_io[i][j].io_rwsem);
sbi->write_io[i][j].sbi = sbi; sbi->write_io[i][j].sbi = sbi;
sbi->write_io[i][j].bio = NULL; sbi->write_io[i][j].bio = NULL;
spin_lock_init(&sbi->write_io[i][j].io_lock);
INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment