NFSv4: Clean up the reboot recovery. Ensure that we exclude stateful

   operations by using a per-server read/write semaphore.
Signed-off-by: Trond Myklebust <trond.myklebust@fys.uio.no>

fs/nfs/inode.c

@@ -1510,8 +1510,13 @@ static int nfs4_fill_super(struct super_block *sb, struct nfs4_mount_data *data,
 		memcpy(clp->cl_ipaddr, server->ip_addr, sizeof(clp->cl_ipaddr));
 		nfs_idmap_new(clp);
 	}
-	if (list_empty(&clp->cl_superblocks))
-		clear_bit(NFS4CLNT_OK, &clp->cl_state);
+	if (list_empty(&clp->cl_superblocks)) {
+		err = nfs4_init_client(clp);
+		if (err != 0) {
+			up_write(&clp->cl_sem);
+			goto out_fail;
+		}
+	}
 	list_add_tail(&server->nfs4_siblings, &clp->cl_superblocks);
 	clnt = rpc_clone_client(clp->cl_rpcclient);
 	if (!IS_ERR(clnt))

fs/nfs/nfs4proc.c

@@ -254,6 +254,7 @@ static int _nfs4_do_open(struct inode *dir, struct qstr *name, int flags, struct
 	struct nfs4_state_owner  *sp;
 	struct nfs4_state     *state = NULL;
 	struct nfs_server       *server = NFS_SERVER(dir);
+	struct nfs4_client *clp = server->nfs4_state;
 	struct inode *inode = NULL;
 	int                     status;
 	struct nfs_fattr        f_attr = {
@@ -279,6 +280,8 @@ static int _nfs4_do_open(struct inode *dir, struct qstr *name, int flags, struct
 		.rpc_cred	= cred,
 	};
 
+	/* Protect against reboot recovery conflicts */
+	down_read(&clp->cl_sem);
 	status = -ENOMEM;
 	if (!(sp = nfs4_get_state_owner(server, cred))) {
 		dprintk("nfs4_do_open: nfs4_get_state_owner failed!\n");
@@ -342,15 +345,18 @@ static int _nfs4_do_open(struct inode *dir, struct qstr *name, int flags, struct
 
 	up(&sp->so_sema);
 	nfs4_put_state_owner(sp);
+	up_read(&clp->cl_sem);
 	*res = state;
 	return 0;
 out_err:
 	if (sp != NULL) {
+		if (state != NULL)
+			nfs4_put_open_state(state);
 		up(&sp->so_sema);
 		nfs4_put_state_owner(sp);
 	}
-	if (state != NULL)
-		nfs4_put_open_state(state);
+	/* Note: clp->cl_sem must be released before nfs4_put_open_state()! */
+	up_read(&clp->cl_sem);
 	if (inode != NULL)
 		iput(inode);
 	*res = NULL;
@@ -446,7 +452,7 @@ int nfs4_do_setattr(struct nfs_server *server, struct nfs_fattr *fattr,
  *
  * NOTE: Caller must be holding the sp->so_owner semaphore!
  */
-int _nfs4_do_close(struct inode *inode, struct nfs4_state *state) 
+static int _nfs4_do_close(struct inode *inode, struct nfs4_state *state) 
 {
 	struct nfs4_state_owner *sp = state->owner;
 	int status = 0;
@@ -475,7 +481,27 @@ int _nfs4_do_close(struct inode *inode, struct nfs4_state *state)
 	return status;
 }
 
-int _nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mode) 
+int nfs4_do_close(struct inode *inode, struct nfs4_state *state) 
+{
+	struct nfs_server *server = NFS_SERVER(state->inode);
+	struct nfs4_exception exception = { };
+	int err;
+	do {
+		err = _nfs4_do_close(inode, state);
+		switch (err) {
+			case -NFS4ERR_STALE_STATEID:
+			case -NFS4ERR_EXPIRED:
+				nfs4_schedule_state_recovery(server->nfs4_state);
+				err = 0;
+			default:
+				state->state = 0;
+		}
+		err = nfs4_handle_exception(server, err, &exception);
+	} while (exception.retry);
+	return err;
+}
+
+static int _nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mode) 
 {
 	struct nfs4_state_owner *sp = state->owner;
 	int status = 0;
@@ -500,6 +526,26 @@ int _nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mod
 	return status;
 }
 
+int nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mode) 
+{
+	struct nfs_server *server = NFS_SERVER(state->inode);
+	struct nfs4_exception exception = { };
+	int err;
+	do {
+		err = _nfs4_do_downgrade(inode, state, mode);
+		switch (err) {
+			case -NFS4ERR_STALE_STATEID:
+			case -NFS4ERR_EXPIRED:
+				nfs4_schedule_state_recovery(server->nfs4_state);
+				err = 0;
+			default:
+				state->state = mode;
+		}
+		err = nfs4_handle_exception(server, err, &exception);
+	} while (exception.retry);
+	return err;
+}
+
 struct inode *
 nfs4_atomic_open(struct inode *dir, struct dentry *dentry, struct nameidata *nd)
 {
@@ -1829,6 +1875,8 @@ nfs4_async_handle_error(struct rpc_task *task, struct nfs_server *server)
 		case -NFS4ERR_EXPIRED:
 			rpc_sleep_on(&clp->cl_rpcwaitq, task, NULL, NULL);
 			nfs4_schedule_state_recovery(clp);
+			if (test_bit(NFS4CLNT_OK, &clp->cl_state))
+				rpc_wake_up_task(task);
 			task->tk_status = 0;
 			return -EAGAIN;
 		case -NFS4ERR_GRACE:
@@ -1844,12 +1892,11 @@ nfs4_async_handle_error(struct rpc_task *task, struct nfs_server *server)
 	return 0;
 }
 
-int
-nfs4_wait_clnt_recover(struct rpc_clnt *clnt, struct nfs4_client *clp)
+int nfs4_wait_clnt_recover(struct rpc_clnt *clnt, struct nfs4_client *clp)
 {
 	DEFINE_WAIT(wait);
 	sigset_t oldset;
-	int interruptible, res;
+	int interruptible, res = 0;
 
 	might_sleep();
 
@@ -1857,19 +1904,12 @@ nfs4_wait_clnt_recover(struct rpc_clnt *clnt, struct nfs4_client *clp)
 	interruptible = TASK_UNINTERRUPTIBLE;
 	if (clnt->cl_intr)
 		interruptible = TASK_INTERRUPTIBLE;
-	do {
-		res = 0;
 	prepare_to_wait(&clp->cl_waitq, &wait, interruptible);
 	nfs4_schedule_state_recovery(clp);
-		if (test_bit(NFS4CLNT_OK, &clp->cl_state) &&
-				!test_bit(NFS4CLNT_SETUP_STATE, &clp->cl_state))
-			break;
-		if (clnt->cl_intr && signalled()) {
+	if (clnt->cl_intr && signalled())
 		res = -ERESTARTSYS;
-			break;
-		}
+	else if (!test_bit(NFS4CLNT_OK, &clp->cl_state))
 		schedule();
-	} while(!test_bit(NFS4CLNT_OK, &clp->cl_state));
 	finish_wait(&clp->cl_waitq, &wait);
 	rpc_clnt_sigunmask(clnt, &oldset);
 	return res;
@@ -2072,6 +2112,7 @@ static int _nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock
 	struct nfs4_lock_state *lsp;
 	int status;
 
+	down_read(&clp->cl_sem);
 	nlo.clientid = clp->cl_clientid;
 	down(&state->lock_sema);
 	lsp = nfs4_find_lock_state(state, request->fl_owner);
@@ -2105,6 +2146,7 @@ static int _nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock
 	if (lsp)
 		nfs4_put_lock_state(lsp);
 	up(&state->lock_sema);
+	up_read(&clp->cl_sem);
 	return status;
 }
 
@@ -2125,6 +2167,7 @@ static int _nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock
 {
 	struct inode *inode = state->inode;
 	struct nfs_server *server = NFS_SERVER(inode);
+	struct nfs4_client *clp = server->nfs4_state;
 	struct nfs_lockargs arg = {
 		.fh = NFS_FH(inode),
 		.type = nfs4_lck_type(cmd, request),
@@ -2144,6 +2187,7 @@ static int _nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock
 	struct nfs_locku_opargs luargs;
 	int status = 0;
 			
+	down_read(&clp->cl_sem);
 	down(&state->lock_sema);
 	lsp = nfs4_find_lock_state(state, request->fl_owner);
 	if (!lsp)
@@ -2164,6 +2208,7 @@ static int _nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock
 	up(&state->lock_sema);
 	if (status == 0)
 		posix_lock_file(request->fl_file, request);
+	up_read(&clp->cl_sem);
 	return status;
 }
 
@@ -2184,6 +2229,7 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock
 {
 	struct inode *inode = state->inode;
 	struct nfs_server *server = NFS_SERVER(inode);
+	struct nfs4_client *clp = server->nfs4_state;
 	struct nfs4_lock_state *lsp;
 	struct nfs_lockargs arg = {
 		.fh = NFS_FH(inode),
@@ -2205,6 +2251,7 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock
 	};
 	int status;
 
+	down_read(&clp->cl_sem);
 	down(&state->lock_sema);
 	lsp = nfs4_find_lock_state(state, request->fl_owner);
 	if (lsp == NULL) {
@@ -2258,6 +2305,7 @@ static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock
 		if (posix_lock_file_wait(request->fl_file, request) < 0)
 			printk(KERN_WARNING "%s: VFS is out of sync with lock manager!\n", __FUNCTION__);
 	}
+	up_read(&clp->cl_sem);
 	return status;
 }
 

include/linux/nfs_fs.h

@@ -507,8 +507,6 @@ struct idmap;
 
 enum nfs4_client_state {
 	NFS4CLNT_OK  = 0,
-	NFS4CLNT_NEW,
-	NFS4CLNT_SETUP_STATE,
 };
 
 /*
@@ -520,7 +518,6 @@ struct nfs4_client {
 	u64			cl_clientid;	/* constant */
 	nfs4_verifier		cl_confirm;
 	unsigned long		cl_state;
-	long			cl_generation;
 
 	u32			cl_lockowner_id;
 
@@ -573,9 +570,7 @@ struct nfs4_state_owner {
 	u32                  so_id;      /* 32-bit identifier, unique */
 	struct semaphore     so_sema;
 	u32                  so_seqid;   /* protected by so_sema */
-	unsigned int         so_flags;   /* protected by so_sema */
 	atomic_t	     so_count;
-	long		     so_generation;
 
 	struct rpc_cred	     *so_cred;	 /* Associated cred */
 	struct list_head     so_states;
@@ -643,8 +638,8 @@ extern int nfs4_proc_setclientid_confirm(struct nfs4_client *);
 extern int nfs4_open_reclaim(struct nfs4_state_owner *, struct nfs4_state *);
 extern int nfs4_proc_async_renew(struct nfs4_client *);
 extern int nfs4_proc_renew(struct nfs4_client *);
-extern int _nfs4_do_close(struct inode *, struct nfs4_state *);
-extern int _nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mode);
+extern int nfs4_do_close(struct inode *, struct nfs4_state *);
+extern int nfs4_do_downgrade(struct inode *inode, struct nfs4_state *state, mode_t mode);
 extern int nfs4_wait_clnt_recover(struct rpc_clnt *, struct nfs4_client *);
 extern struct inode *nfs4_atomic_open(struct inode *, struct dentry *, struct nameidata *);
 extern int nfs4_open_revalidate(struct inode *, struct dentry *, int);
@@ -660,6 +655,7 @@ extern void init_nfsv4_state(struct nfs_server *);
 extern void destroy_nfsv4_state(struct nfs_server *);
 extern struct nfs4_client *nfs4_get_client(struct in_addr *);
 extern void nfs4_put_client(struct nfs4_client *clp);
+extern int nfs4_init_client(struct nfs4_client *clp);
 extern u32 nfs4_alloc_lockowner_id(struct nfs4_client *);
 
 extern struct nfs4_state_owner * nfs4_get_state_owner(struct nfs_server *, struct rpc_cred *);