Commit 252470cd authored by Trond Myklebust's avatar Trond Myklebust

NFS: Break the nfs_wreq_lock into per-mount locks. This helps prevent

   a heavy read and write workload on one mount point from
   interfering with workloads on other mount points.

Note that there is still some serialization due to the big kernel
   lock.
Signed-off-by: default avatarChuck Lever <cel@netapp.com>
Signed-off-by: default avatarTrond Myklebust <trond.myklebust@fys.uio.no>
parent 395f639e
...@@ -1764,6 +1764,7 @@ static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags) ...@@ -1764,6 +1764,7 @@ static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags)
if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
SLAB_CTOR_CONSTRUCTOR) { SLAB_CTOR_CONSTRUCTOR) {
inode_init_once(&nfsi->vfs_inode); inode_init_once(&nfsi->vfs_inode);
spin_lock_init(&nfsi->req_lock);
INIT_LIST_HEAD(&nfsi->dirty); INIT_LIST_HEAD(&nfsi->dirty);
INIT_LIST_HEAD(&nfsi->commit); INIT_LIST_HEAD(&nfsi->commit);
INIT_RADIX_TREE(&nfsi->nfs_page_tree, GFP_ATOMIC); INIT_RADIX_TREE(&nfsi->nfs_page_tree, GFP_ATOMIC);
......
...@@ -21,11 +21,6 @@ ...@@ -21,11 +21,6 @@
#define NFS_PARANOIA 1 #define NFS_PARANOIA 1
/*
* Spinlock
*/
spinlock_t nfs_wreq_lock = SPIN_LOCK_UNLOCKED;
static kmem_cache_t *nfs_page_cachep; static kmem_cache_t *nfs_page_cachep;
static inline struct nfs_page * static inline struct nfs_page *
...@@ -95,7 +90,7 @@ nfs_create_request(struct file *file, struct inode *inode, ...@@ -95,7 +90,7 @@ nfs_create_request(struct file *file, struct inode *inode,
req->wb_pgbase = offset; req->wb_pgbase = offset;
req->wb_bytes = count; req->wb_bytes = count;
req->wb_inode = inode; req->wb_inode = inode;
req->wb_count = 1; atomic_set(&req->wb_count, 1);
server->rpc_ops->request_init(req, file); server->rpc_ops->request_init(req, file);
return req; return req;
...@@ -137,12 +132,8 @@ void nfs_clear_request(struct nfs_page *req) ...@@ -137,12 +132,8 @@ void nfs_clear_request(struct nfs_page *req)
void void
nfs_release_request(struct nfs_page *req) nfs_release_request(struct nfs_page *req)
{ {
spin_lock(&nfs_wreq_lock); if (!atomic_dec_and_test(&req->wb_count))
if (--req->wb_count) {
spin_unlock(&nfs_wreq_lock);
return; return;
}
spin_unlock(&nfs_wreq_lock);
#ifdef NFS_PARANOIA #ifdef NFS_PARANOIA
BUG_ON (!list_empty(&req->wb_list)); BUG_ON (!list_empty(&req->wb_list));
...@@ -254,7 +245,7 @@ nfs_coalesce_requests(struct list_head *head, struct list_head *dst, ...@@ -254,7 +245,7 @@ nfs_coalesce_requests(struct list_head *head, struct list_head *dst,
* If the number of requests is set to 0, the entire address_space * If the number of requests is set to 0, the entire address_space
* starting at index idx_start, is scanned. * starting at index idx_start, is scanned.
* The requests are *not* checked to ensure that they form a contiguous set. * The requests are *not* checked to ensure that they form a contiguous set.
* You must be holding the nfs_wreq_lock when calling this function * You must be holding the inode's req_lock when calling this function
*/ */
int int
nfs_scan_list(struct list_head *head, struct list_head *dst, nfs_scan_list(struct list_head *head, struct list_head *dst,
......
...@@ -389,7 +389,7 @@ nfs_inode_add_request(struct inode *inode, struct nfs_page *req) ...@@ -389,7 +389,7 @@ nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
nfs_begin_data_update(inode); nfs_begin_data_update(inode);
} }
nfsi->npages++; nfsi->npages++;
req->wb_count++; atomic_inc(&req->wb_count);
return 0; return 0;
} }
...@@ -399,21 +399,20 @@ nfs_inode_add_request(struct inode *inode, struct nfs_page *req) ...@@ -399,21 +399,20 @@ nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
static void static void
nfs_inode_remove_request(struct nfs_page *req) nfs_inode_remove_request(struct nfs_page *req)
{ {
struct nfs_inode *nfsi; struct inode *inode = req->wb_inode;
struct inode *inode; struct nfs_inode *nfsi = NFS_I(inode);
BUG_ON (!NFS_WBACK_BUSY(req)); BUG_ON (!NFS_WBACK_BUSY(req));
spin_lock(&nfs_wreq_lock);
inode = req->wb_inode; spin_lock(&nfsi->req_lock);
nfsi = NFS_I(inode);
radix_tree_delete(&nfsi->nfs_page_tree, req->wb_index); radix_tree_delete(&nfsi->nfs_page_tree, req->wb_index);
nfsi->npages--; nfsi->npages--;
if (!nfsi->npages) { if (!nfsi->npages) {
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
nfs_end_data_update_defer(inode); nfs_end_data_update_defer(inode);
iput(inode); iput(inode);
} else } else
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
nfs_clear_request(req); nfs_clear_request(req);
nfs_release_request(req); nfs_release_request(req);
} }
...@@ -429,7 +428,7 @@ _nfs_find_request(struct inode *inode, unsigned long index) ...@@ -429,7 +428,7 @@ _nfs_find_request(struct inode *inode, unsigned long index)
req = (struct nfs_page*)radix_tree_lookup(&nfsi->nfs_page_tree, index); req = (struct nfs_page*)radix_tree_lookup(&nfsi->nfs_page_tree, index);
if (req) if (req)
req->wb_count++; atomic_inc(&req->wb_count);
return req; return req;
} }
...@@ -437,10 +436,11 @@ static struct nfs_page * ...@@ -437,10 +436,11 @@ static struct nfs_page *
nfs_find_request(struct inode *inode, unsigned long index) nfs_find_request(struct inode *inode, unsigned long index)
{ {
struct nfs_page *req; struct nfs_page *req;
struct nfs_inode *nfsi = NFS_I(inode);
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
req = _nfs_find_request(inode, index); req = _nfs_find_request(inode, index);
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
return req; return req;
} }
...@@ -453,10 +453,10 @@ nfs_mark_request_dirty(struct nfs_page *req) ...@@ -453,10 +453,10 @@ nfs_mark_request_dirty(struct nfs_page *req)
struct inode *inode = req->wb_inode; struct inode *inode = req->wb_inode;
struct nfs_inode *nfsi = NFS_I(inode); struct nfs_inode *nfsi = NFS_I(inode);
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
nfs_list_add_request(req, &nfsi->dirty); nfs_list_add_request(req, &nfsi->dirty);
nfsi->ndirty++; nfsi->ndirty++;
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
inc_page_state(nr_dirty); inc_page_state(nr_dirty);
mark_inode_dirty(inode); mark_inode_dirty(inode);
} }
...@@ -481,10 +481,10 @@ nfs_mark_request_commit(struct nfs_page *req) ...@@ -481,10 +481,10 @@ nfs_mark_request_commit(struct nfs_page *req)
struct inode *inode = req->wb_inode; struct inode *inode = req->wb_inode;
struct nfs_inode *nfsi = NFS_I(inode); struct nfs_inode *nfsi = NFS_I(inode);
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
nfs_list_add_request(req, &nfsi->commit); nfs_list_add_request(req, &nfsi->commit);
nfsi->ncommit++; nfsi->ncommit++;
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
inc_page_state(nr_unstable); inc_page_state(nr_unstable);
mark_inode_dirty(inode); mark_inode_dirty(inode);
} }
...@@ -509,7 +509,7 @@ nfs_wait_on_requests(struct inode *inode, unsigned long idx_start, unsigned int ...@@ -509,7 +509,7 @@ nfs_wait_on_requests(struct inode *inode, unsigned long idx_start, unsigned int
else else
idx_end = idx_start + npages - 1; idx_end = idx_start + npages - 1;
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
next = idx_start; next = idx_start;
while (radix_tree_gang_lookup(&nfsi->nfs_page_tree, (void **)&req, next, 1)) { while (radix_tree_gang_lookup(&nfsi->nfs_page_tree, (void **)&req, next, 1)) {
if (req->wb_index > idx_end) if (req->wb_index > idx_end)
...@@ -519,16 +519,16 @@ nfs_wait_on_requests(struct inode *inode, unsigned long idx_start, unsigned int ...@@ -519,16 +519,16 @@ nfs_wait_on_requests(struct inode *inode, unsigned long idx_start, unsigned int
if (!NFS_WBACK_BUSY(req)) if (!NFS_WBACK_BUSY(req))
continue; continue;
req->wb_count++; atomic_inc(&req->wb_count);
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
error = nfs_wait_on_request(req); error = nfs_wait_on_request(req);
nfs_release_request(req); nfs_release_request(req);
if (error < 0) if (error < 0)
return error; return error;
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
res++; res++;
} }
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
return res; return res;
} }
...@@ -624,6 +624,7 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page, ...@@ -624,6 +624,7 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page,
unsigned int offset, unsigned int bytes) unsigned int offset, unsigned int bytes)
{ {
struct nfs_server *server = NFS_SERVER(inode); struct nfs_server *server = NFS_SERVER(inode);
struct nfs_inode *nfsi = NFS_I(inode);
struct nfs_page *req, *new = NULL; struct nfs_page *req, *new = NULL;
unsigned long rqend, end; unsigned long rqend, end;
...@@ -635,19 +636,19 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page, ...@@ -635,19 +636,19 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page,
/* Loop over all inode entries and see if we find /* Loop over all inode entries and see if we find
* A request for the page we wish to update * A request for the page we wish to update
*/ */
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
req = _nfs_find_request(inode, page->index); req = _nfs_find_request(inode, page->index);
if (req) { if (req) {
if (!nfs_lock_request_dontget(req)) { if (!nfs_lock_request_dontget(req)) {
int error; int error;
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
error = nfs_wait_on_request(req); error = nfs_wait_on_request(req);
nfs_release_request(req); nfs_release_request(req);
if (error < 0) if (error < 0)
return ERR_PTR(error); return ERR_PTR(error);
continue; continue;
} }
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
if (new) if (new)
nfs_release_request(new); nfs_release_request(new);
break; break;
...@@ -658,15 +659,15 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page, ...@@ -658,15 +659,15 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page,
nfs_lock_request_dontget(new); nfs_lock_request_dontget(new);
error = nfs_inode_add_request(inode, new); error = nfs_inode_add_request(inode, new);
if (error) { if (error) {
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
nfs_unlock_request(new); nfs_unlock_request(new);
return ERR_PTR(error); return ERR_PTR(error);
} }
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
nfs_mark_request_dirty(new); nfs_mark_request_dirty(new);
return new; return new;
} }
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
new = nfs_create_request(file, inode, page, offset, bytes); new = nfs_create_request(file, inode, page, offset, bytes);
if (IS_ERR(new)) if (IS_ERR(new))
...@@ -1347,13 +1348,14 @@ nfs_commit_done(struct rpc_task *task) ...@@ -1347,13 +1348,14 @@ nfs_commit_done(struct rpc_task *task)
int nfs_flush_inode(struct inode *inode, unsigned long idx_start, int nfs_flush_inode(struct inode *inode, unsigned long idx_start,
unsigned int npages, int how) unsigned int npages, int how)
{ {
struct nfs_inode *nfsi = NFS_I(inode);
LIST_HEAD(head); LIST_HEAD(head);
int res, int res,
error = 0; error = 0;
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
res = nfs_scan_dirty(inode, &head, idx_start, npages); res = nfs_scan_dirty(inode, &head, idx_start, npages);
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
if (res) if (res)
error = nfs_flush_list(&head, NFS_SERVER(inode)->wpages, how); error = nfs_flush_list(&head, NFS_SERVER(inode)->wpages, how);
if (error < 0) if (error < 0)
...@@ -1365,18 +1367,19 @@ int nfs_flush_inode(struct inode *inode, unsigned long idx_start, ...@@ -1365,18 +1367,19 @@ int nfs_flush_inode(struct inode *inode, unsigned long idx_start,
int nfs_commit_inode(struct inode *inode, unsigned long idx_start, int nfs_commit_inode(struct inode *inode, unsigned long idx_start,
unsigned int npages, int how) unsigned int npages, int how)
{ {
struct nfs_inode *nfsi = NFS_I(inode);
LIST_HEAD(head); LIST_HEAD(head);
int res, int res,
error = 0; error = 0;
spin_lock(&nfs_wreq_lock); spin_lock(&nfsi->req_lock);
res = nfs_scan_commit(inode, &head, idx_start, npages); res = nfs_scan_commit(inode, &head, idx_start, npages);
if (res) { if (res) {
res += nfs_scan_commit(inode, &head, 0, 0); res += nfs_scan_commit(inode, &head, 0, 0);
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
error = nfs_commit_list(&head, how); error = nfs_commit_list(&head, how);
} else } else
spin_unlock(&nfs_wreq_lock); spin_unlock(&nfsi->req_lock);
if (error < 0) if (error < 0)
return error; return error;
return res; return res;
......
...@@ -147,6 +147,7 @@ struct nfs_inode { ...@@ -147,6 +147,7 @@ struct nfs_inode {
/* /*
* This is the list of dirty unwritten pages. * This is the list of dirty unwritten pages.
*/ */
spinlock_t req_lock;
struct list_head dirty; struct list_head dirty;
struct list_head commit; struct list_head commit;
struct radix_tree_root nfs_page_tree; struct radix_tree_root nfs_page_tree;
......
...@@ -40,8 +40,8 @@ struct nfs_page { ...@@ -40,8 +40,8 @@ struct nfs_page {
unsigned long wb_index; /* Offset >> PAGE_CACHE_SHIFT */ unsigned long wb_index; /* Offset >> PAGE_CACHE_SHIFT */
unsigned int wb_offset, /* Offset & ~PAGE_CACHE_MASK */ unsigned int wb_offset, /* Offset & ~PAGE_CACHE_MASK */
wb_pgbase, /* Start of page data */ wb_pgbase, /* Start of page data */
wb_bytes, /* Length of request */ wb_bytes; /* Length of request */
wb_count; /* reference count */ atomic_t wb_count; /* reference count */
unsigned long wb_flags; unsigned long wb_flags;
struct nfs_writeverf wb_verf; /* Commit cookie */ struct nfs_writeverf wb_verf; /* Commit cookie */
}; };
...@@ -65,8 +65,6 @@ extern int nfs_coalesce_requests(struct list_head *, struct list_head *, ...@@ -65,8 +65,6 @@ extern int nfs_coalesce_requests(struct list_head *, struct list_head *,
unsigned int); unsigned int);
extern int nfs_wait_on_request(struct nfs_page *); extern int nfs_wait_on_request(struct nfs_page *);
extern spinlock_t nfs_wreq_lock;
/* /*
* Lock the page of an asynchronous request without incrementing the wb_count * Lock the page of an asynchronous request without incrementing the wb_count
*/ */
...@@ -86,7 +84,7 @@ nfs_lock_request(struct nfs_page *req) ...@@ -86,7 +84,7 @@ nfs_lock_request(struct nfs_page *req)
{ {
if (test_and_set_bit(PG_BUSY, &req->wb_flags)) if (test_and_set_bit(PG_BUSY, &req->wb_flags))
return 0; return 0;
req->wb_count++; atomic_inc(&req->wb_count);
return 1; return 1;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment