NFSv4: Basic code for managing delegation state.

Signed-off-by: Trond Myklebust <trond.myklebust@fys.uio.no>

fs/nfs/Makefile

@@ -9,6 +9,6 @@ nfs-y 			:= dir.o file.o inode.o nfs2xdr.o pagelist.o \
 nfs-$(CONFIG_ROOT_NFS)	+= nfsroot.o mount_clnt.o      
 nfs-$(CONFIG_NFS_V3)	+= nfs3proc.o nfs3xdr.o
 nfs-$(CONFIG_NFS_V4)	+= nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o \
-			   idmap.o
+			   delegation.o idmap.o
 nfs-$(CONFIG_NFS_DIRECTIO) += direct.o
 nfs-objs		:= $(nfs-y)

fs/nfs/delegation.c

+/*
+ * linux/fs/nfs/delegation.c
+ *
+ * Copyright (C) 2004 Trond Myklebust
+ *
+ * NFS file delegation management
+ *
+ */
+#include <linux/nfs4.h>
+#include <linux/nfs_fs.h>
+#include <linux/nfs_xdr.h>
+#include <linux/spinlock.h>
+
+#include "delegation.h"
+
+static struct nfs_delegation *nfs_alloc_delegation(void)
+{
+	return (struct nfs_delegation *)kmalloc(sizeof(struct nfs_delegation), GFP_KERNEL);
+}
+
+static void nfs_free_delegation(struct nfs_delegation *delegation)
+{
+	if (delegation->cred)
+		put_rpccred(delegation->cred);
+	kfree(delegation);
+}
+
+/*
+ * Set up a delegation on an inode
+ */
+int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct nfs_openres *res)
+{
+	struct nfs4_client *clp = NFS_SERVER(inode)->nfs4_state;
+	struct nfs_inode *nfsi = NFS_I(inode);
+	struct nfs_delegation *delegation;
+	int status = 0;
+
+	delegation = nfs_alloc_delegation();
+	if (delegation == NULL)
+		return -ENOMEM;
+	memcpy(delegation->stateid.data, res->delegation.data,
+			sizeof(delegation->stateid.data));
+	delegation->type = res->delegation_type;
+	delegation->maxsize = res->maxsize;
+	delegation->cred = get_rpccred(cred);
+	delegation->inode = inode;
+
+	spin_lock(&clp->cl_lock);
+	if (nfsi->delegation == NULL) {
+		list_add(&delegation->super_list, &clp->cl_delegations);
+		nfsi->delegation = delegation;
+		delegation = NULL;
+	} else {
+		if (memcmp(&delegation->stateid, &nfsi->delegation->stateid,
+					sizeof(delegation->stateid)) != 0 ||
+				delegation->type != nfsi->delegation->type) {
+			printk("%s: server %u.%u.%u.%u, handed out a duplicate delegation!\n",
+					__FUNCTION__, NIPQUAD(clp->cl_addr));
+			status = -EIO;
+		}
+	}
+	spin_unlock(&clp->cl_lock);
+	if (delegation != NULL)
+		kfree(delegation);
+	return status;
+}
+
+static int nfs_do_return_delegation(struct inode *inode, struct nfs_delegation *delegation)
+{
+	int res = 0;
+
+	__nfs_revalidate_inode(NFS_SERVER(inode), inode);
+
+	res = nfs4_proc_delegreturn(inode, delegation->cred, &delegation->stateid);
+	nfs_free_delegation(delegation);
+	return res;
+}
+
+/* Sync all data to disk upon delegation return */
+static void nfs_msync_inode(struct inode *inode)
+{
+	filemap_fdatawrite(inode->i_mapping);
+	nfs_wb_all(inode);
+	filemap_fdatawait(inode->i_mapping);
+}
+
+/*
+ * Basic procedure for returning a delegation to the server
+ */
+int nfs_inode_return_delegation(struct inode *inode)
+{
+	struct nfs4_client *clp = NFS_SERVER(inode)->nfs4_state;
+	struct nfs_inode *nfsi = NFS_I(inode);
+	struct nfs_delegation *delegation;
+	int res = 0;
+
+	nfs_msync_inode(inode);
+	down_read(&clp->cl_sem);
+	/* Guard against new delegated open calls */
+	down_write(&nfsi->rwsem);
+	spin_lock(&clp->cl_lock);
+	delegation = nfsi->delegation;
+	if (delegation != NULL) {
+		list_del_init(&delegation->super_list);
+		nfsi->delegation = NULL;
+	}
+	spin_unlock(&clp->cl_lock);
+	up_write(&nfsi->rwsem);
+	up_read(&clp->cl_sem);
+	nfs_msync_inode(inode);
+
+	if (delegation != NULL)
+		res = nfs_do_return_delegation(inode, delegation);
+	return res;
+}
+
+/*
+ * Return all delegations associated to a super block
+ */
+void nfs_return_all_delegations(struct super_block *sb)
+{
+	struct nfs4_client *clp = NFS_SB(sb)->nfs4_state;
+	struct nfs_delegation *delegation;
+	struct inode *inode;
+
+	if (clp == NULL)
+		return;
+restart:
+	spin_lock(&clp->cl_lock);
+	list_for_each_entry(delegation, &clp->cl_delegations, super_list) {
+		if (delegation->inode->i_sb != sb)
+			continue;
+		inode = igrab(delegation->inode);
+		if (inode == NULL)
+			continue;
+		spin_unlock(&clp->cl_lock);
+		nfs_inode_return_delegation(inode);
+		iput(inode);
+		goto restart;
+	}
+	spin_unlock(&clp->cl_lock);
+}
+
+/*
+ * Retrieve the inode associated with a delegation
+ */
+struct inode *nfs_delegation_find_inode(struct nfs4_client *clp, const struct nfs_fh *fhandle)
+{
+	struct nfs_delegation *delegation;
+	struct inode *res = NULL;
+	spin_lock(&clp->cl_lock);
+	list_for_each_entry(delegation, &clp->cl_delegations, super_list) {
+		if (nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) {
+			res = igrab(delegation->inode);
+			break;
+		}
+	}
+	spin_unlock(&clp->cl_lock);
+	return res;
+}

fs/nfs/delegation.h

+/*
+ * linux/fs/nfs/delegation.h
+ *
+ * Copyright (c) Trond Myklebust
+ *
+ * Definitions pertaining to NFS delegated files
+ */
+#ifndef FS_NFS_DELEGATION_H
+#define FS_NFS_DELEGATION_H
+
+/*
+ * NFSv4 delegation
+ */
+struct nfs_delegation {
+	struct list_head super_list;
+	struct rpc_cred *cred;
+	struct inode *inode;
+	nfs4_stateid stateid;
+	int type;
+	loff_t maxsize;
+};
+
+int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct nfs_openres *res);
+int nfs_inode_return_delegation(struct inode *inode);
+
+struct inode *nfs_delegation_find_inode(struct nfs4_client *clp, const struct nfs_fh *fhandle);
+void nfs_return_all_delegations(struct super_block *sb);
+/* NFSv4 delegation-related procedures */
+int nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, const nfs4_stateid *stateid);
+
+#endif

fs/nfs/inode.c

@@ -1437,8 +1437,11 @@ static struct file_system_type nfs_fs_type = {
 
 #ifdef CONFIG_NFS_V4
 
+#include "delegation.h"
+
 static void nfs4_clear_inode(struct inode *);
 
+
 static struct super_operations nfs4_sops = { 
 	.alloc_inode	= nfs_alloc_inode,
 	.destroy_inode	= nfs_destroy_inode,
@@ -1459,6 +1462,9 @@ static void nfs4_clear_inode(struct inode *inode)
 {
 	struct nfs_inode *nfsi = NFS_I(inode);
 
+	/* If we are holding a delegation, return it! */
+	if (nfsi->delegation != NULL)
+		nfs_inode_return_delegation(inode);
 	/* First call standard NFS clear_inode() code */
 	nfs_clear_inode(inode);
 	/* Now clear out any remaining state */
@@ -1743,22 +1749,30 @@ static struct super_block *nfs4_get_sb(struct file_system_type *fs_type,
 	return s;
 }
 
+static void nfs4_kill_super(struct super_block *sb)
+{
+	nfs_return_all_delegations(sb);
+	nfs_kill_super(sb);
+}
+
 static struct file_system_type nfs4_fs_type = {
 	.owner		= THIS_MODULE,
 	.name		= "nfs4",
 	.get_sb		= nfs4_get_sb,
-	.kill_sb	= nfs_kill_super,
+	.kill_sb	= nfs4_kill_super,
 	.fs_flags	= FS_ODD_RENAME|FS_REVAL_DOT|FS_BINARY_MOUNTDATA,
 };
 
-#define nfs4_zero_state(nfsi) \
+#define nfs4_init_once(nfsi) \
 	do { \
 		INIT_LIST_HEAD(&(nfsi)->open_states); \
+		nfsi->delegation = NULL; \
+		init_rwsem(&nfsi->rwsem); \
 	} while(0)
 #define register_nfs4fs() register_filesystem(&nfs4_fs_type)
 #define unregister_nfs4fs() unregister_filesystem(&nfs4_fs_type)
 #else
-#define nfs4_zero_state(nfsi) \
+#define nfs4_init_once(nfsi) \
 	do { } while (0)
 #define register_nfs4fs() (0)
 #define unregister_nfs4fs()
@@ -1780,7 +1794,6 @@ static struct inode *nfs_alloc_inode(struct super_block *sb)
 	if (!nfsi)
 		return NULL;
 	nfsi->flags = 0;
-	nfs4_zero_state(nfsi);
 	return &nfsi->vfs_inode;
 }
 
@@ -1806,6 +1819,7 @@ static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags)
 		nfsi->ncommit = 0;
 		nfsi->npages = 0;
 		init_waitqueue_head(&nfsi->nfs_i_wait);
+		nfs4_init_once(nfsi);
 	}
 }
  

fs/nfs/nfs4proc.c

@@ -47,6 +47,8 @@
 #include <linux/smp_lock.h>
 #include <linux/namei.h>
 
+#include "delegation.h"
+
 #define NFSDBG_FACILITY		NFSDBG_PROC
 
 #define NFS4_POLL_RETRY_MIN	(1*HZ)
@@ -298,7 +300,7 @@ static int _nfs4_do_open(struct inode *dir, struct qstr *name, int flags, struct
 	down(&sp->so_sema);
 	o_arg.seqid = sp->so_seqid;
 	o_arg.id = sp->so_id;
-	o_arg.clientid = NFS_SERVER(dir)->nfs4_state->cl_clientid,
+	o_arg.clientid = clp->cl_clientid,
 
 	status = rpc_call_sync(server->client, &msg, 0);
 	nfs4_increment_seqid(status, sp);
@@ -347,7 +349,8 @@ static int _nfs4_do_open(struct inode *dir, struct qstr *name, int flags, struct
 		state->nwriters++;
 	state->state |= flags & (FMODE_READ|FMODE_WRITE);
 	spin_unlock(&inode->i_lock);
-
+	if (o_res.delegation_type != 0)
+		nfs_inode_set_delegation(inode, cred, &o_res);
 	up(&sp->so_sema);
 	nfs4_put_state_owner(sp);
 	up_read(&clp->cl_sem);
@@ -1993,6 +1996,40 @@ nfs4_proc_setclientid_confirm(struct nfs4_client *clp)
 	return status;
 }
 
+static int _nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, const nfs4_stateid *stateid)
+{
+	struct nfs4_delegreturnargs args = {
+		.fhandle = NFS_FH(inode),
+		.stateid = stateid,
+	};
+	struct rpc_message msg = {
+		.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DELEGRETURN],
+		.rpc_argp = &args,
+		.rpc_cred = cred,
+	};
+
+	return rpc_call_sync(NFS_CLIENT(inode), &msg, 0);
+}
+
+int nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, const nfs4_stateid *stateid)
+{
+	struct nfs_server *server = NFS_SERVER(inode);
+	struct nfs4_exception exception = { };
+	int err;
+	do {
+		err = _nfs4_proc_delegreturn(inode, cred, stateid);
+		switch (err) {
+			case -NFS4ERR_STALE_STATEID:
+			case -NFS4ERR_EXPIRED:
+				nfs4_schedule_state_recovery(server->nfs4_state);
+			case 0:
+				return 0;
+		}
+		err = nfs4_handle_exception(server, err, &exception);
+	} while (exception.retry);
+	return err;
+}
+
 #define NFS4_LOCK_MINTIMEOUT (1 * HZ)
 #define NFS4_LOCK_MAXTIMEOUT (30 * HZ)
 

fs/nfs/nfs4state.c

@@ -98,6 +98,7 @@ nfs4_alloc_client(struct in_addr *addr)
 		memset(clp, 0, sizeof(*clp));
 		memcpy(&clp->cl_addr, addr, sizeof(clp->cl_addr));
 		init_rwsem(&clp->cl_sem);
+		INIT_LIST_HEAD(&clp->cl_delegations);
 		INIT_LIST_HEAD(&clp->cl_state_owners);
 		INIT_LIST_HEAD(&clp->cl_unused);
 		spin_lock_init(&clp->cl_lock);
@@ -234,6 +235,7 @@ nfs4_alloc_state_owner(void)
 	init_MUTEX(&sp->so_sema);
 	sp->so_seqid = 0;                 /* arbitrary */
 	INIT_LIST_HEAD(&sp->so_states);
+	INIT_LIST_HEAD(&sp->so_delegations);
 	atomic_set(&sp->so_count, 1);
 	return sp;
 }

fs/nfs/nfs4xdr.c

@@ -128,6 +128,8 @@ static int nfs_stat_to_errno(int);
 				2 + nfs4_name_maxsz + \
 				nfs4_fattr_bitmap_maxsz)
 #define decode_create_maxsz	(op_decode_hdr_maxsz + 8)
+#define encode_delegreturn_maxsz (op_encode_hdr_maxsz + 4)
+#define decode_delegreturn_maxsz (op_decode_hdr_maxsz)
 #define NFS4_enc_compound_sz	(1024)  /* XXX: large enough? */
 #define NFS4_dec_compound_sz	(1024)  /* XXX: large enough? */
 #define NFS4_enc_read_sz	(compound_encode_hdr_maxsz + \
@@ -355,6 +357,11 @@ static int nfs_stat_to_errno(int);
 				encode_getattr_maxsz)
 #define NFS4_dec_server_caps_sz (compound_decode_hdr_maxsz + \
 				decode_getattr_maxsz)
+#define NFS4_enc_delegreturn_sz	(compound_encode_hdr_maxsz + \
+				encode_putfh_maxsz + \
+				encode_delegreturn_maxsz)
+#define NFS4_dec_delegreturn_sz (compound_decode_hdr_maxsz + \
+				decode_delegreturn_maxsz)
 
 static struct {
 	unsigned int	mode;
@@ -1103,6 +1110,18 @@ static int encode_write(struct xdr_stream *xdr, const struct nfs_writeargs *args
 
 	return 0;
 }
+
+static int encode_delegreturn(struct xdr_stream *xdr, const nfs4_stateid *stateid)
+{
+	uint32_t *p;
+
+	RESERVE_SPACE(20);
+
+	WRITE32(OP_DELEGRETURN);
+	WRITEMEM(stateid->data, sizeof(stateid->data));
+	return 0;
+
+}
 /*
  * END OF "GENERIC" ENCODE ROUTINES.
  */
@@ -1740,6 +1759,24 @@ static int nfs4_xdr_enc_setclientid_confirm(struct rpc_rqst *req, uint32_t *p, s
 	return status;
 }
 
+/*
+ * DELEGRETURN request
+ */
+static int nfs4_xdr_enc_delegreturn(struct rpc_rqst *req, uint32_t *p, const struct nfs4_delegreturnargs *args)
+{
+	struct xdr_stream xdr;
+	struct compound_hdr hdr = {
+		.nops = 2,
+	};
+	int status;
+
+	xdr_init_encode(&xdr, &req->rq_snd_buf, p);
+	encode_compound_hdr(&xdr, &hdr);
+	if ((status = encode_putfh(&xdr, args->fhandle)) == 0)
+		status = encode_delegreturn(&xdr, args->stateid);
+	return status;
+}
+
 /*
  * START OF "GENERIC" DECODE ROUTINES.
  *   These may look a little ugly since they are imported from a "generic"
@@ -1774,6 +1811,17 @@ static int nfs4_xdr_enc_setclientid_confirm(struct rpc_rqst *req, uint32_t *p, s
 	} \
 } while (0)
 
+static int decode_opaque_inline(struct xdr_stream *xdr, uint32_t *len, char **string)
+{
+	uint32_t *p;
+
+	READ_BUF(4);
+	READ32(*len);
+	READ_BUF(*len);
+	*string = (char *)p;
+	return 0;
+}
+
 static int decode_compound_hdr(struct xdr_stream *xdr, struct compound_hdr *hdr)
 {
 	uint32_t *p;
@@ -1810,6 +1858,17 @@ static int decode_op_hdr(struct xdr_stream *xdr, enum nfs_opnum4 expected)
 	return 0;
 }
 
+/* Dummy routine */
+static int decode_ace(struct xdr_stream *xdr, void *ace, struct nfs4_client *clp)
+{
+	uint32_t *p;
+	uint32_t strlen;
+	char *str;
+
+	READ_BUF(12);
+	return decode_opaque_inline(xdr, &strlen, &str);
+}
+
 static int decode_attr_bitmap(struct xdr_stream *xdr, uint32_t *bitmap)
 {
 	uint32_t bmlen, *p;
@@ -2742,10 +2801,56 @@ static int decode_lookup(struct xdr_stream *xdr)
 	return decode_op_hdr(xdr, OP_LOOKUP);
 }
 
+/* This is too sick! */
+static int decode_space_limit(struct xdr_stream *xdr, u64 *maxsize)
+{
+        uint32_t *p;
+	uint32_t limit_type, nblocks, blocksize;
+
+	READ_BUF(12);
+	READ32(limit_type);
+	switch (limit_type) {
+		case 1:
+			READ64(*maxsize);
+			break;
+		case 2:
+			READ32(nblocks);
+			READ32(blocksize);
+			*maxsize = (uint64_t)nblocks * (uint64_t)blocksize;
+	}
+	return 0;
+}
+
+static int decode_delegation(struct xdr_stream *xdr, struct nfs_openres *res)
+{
+        uint32_t *p;
+        uint32_t delegation_type;
+
+	READ_BUF(4);
+	READ32(delegation_type);
+	if (delegation_type == NFS4_OPEN_DELEGATE_NONE) {
+		res->delegation_type = 0;
+		return 0;
+	}
+	READ_BUF(20);
+	COPYMEM(res->delegation.data, sizeof(res->delegation.data));
+	READ32(res->do_recall);
+	switch (delegation_type) {
+		case NFS4_OPEN_DELEGATE_READ:
+			res->delegation_type = FMODE_READ;
+			break;
+		case NFS4_OPEN_DELEGATE_WRITE:
+			res->delegation_type = FMODE_WRITE|FMODE_READ;
+			if (decode_space_limit(xdr, &res->maxsize) < 0)
+				return -EIO;
+	}
+	return decode_ace(xdr, NULL, res->server->nfs4_state);
+}
+
 static int decode_open(struct xdr_stream *xdr, struct nfs_openres *res)
 {
         uint32_t *p;
-        uint32_t bmlen, delegation_type;
+        uint32_t bmlen;
         int status;
 
         status = decode_op_hdr(xdr, OP_OPEN);
@@ -2762,11 +2867,9 @@ static int decode_open(struct xdr_stream *xdr, struct nfs_openres *res)
         if (bmlen > 10)
                 goto xdr_error;
 
-        READ_BUF((bmlen << 2) + 4);
+        READ_BUF(bmlen << 2);
         p += bmlen;
-        READ32(delegation_type);
-        if (delegation_type == NFS4_OPEN_DELEGATE_NONE)
-		return 0;
+	return decode_delegation(xdr, res);
 xdr_error:
 	printk(KERN_NOTICE "%s: xdr error!\n", __FUNCTION__);
 	return -EIO;
@@ -3055,6 +3158,11 @@ static int decode_write(struct xdr_stream *xdr, struct nfs_writeres *res)
 	return 0;
 }
 
+static int decode_delegreturn(struct xdr_stream *xdr)
+{
+	return decode_op_hdr(xdr, OP_DELEGRETURN);
+}
+
 /*
  * Decode OPEN_DOWNGRADE response
  */
@@ -3688,6 +3796,25 @@ static int nfs4_xdr_dec_setclientid_confirm(struct rpc_rqst *req, uint32_t *p, s
 	return status;
 }
 
+/*
+ * DELEGRETURN request
+ */
+static int nfs4_xdr_dec_delegreturn(struct rpc_rqst *rqstp, uint32_t *p, void *dummy)
+{
+	struct xdr_stream xdr;
+	struct compound_hdr hdr;
+	int status;
+
+	xdr_init_decode(&xdr, &rqstp->rq_rcv_buf, p);
+	status = decode_compound_hdr(&xdr, &hdr);
+	if (status == 0) {
+		status = decode_putfh(&xdr);
+		if (status == 0)
+			status = decode_delegreturn(&xdr);
+	}
+	return status;
+}
+
 uint32_t *nfs4_decode_dirent(uint32_t *p, struct nfs_entry *entry, int plus)
 {
 	uint32_t bitmap[1] = {0};
@@ -3841,6 +3968,7 @@ struct rpc_procinfo	nfs4_procedures[] = {
   PROC(READLINK,	enc_readlink,	dec_readlink),
   PROC(READDIR,		enc_readdir,	dec_readdir),
   PROC(SERVER_CAPS,	enc_server_caps, dec_server_caps),
+  PROC(DELEGRETURN,	enc_delegreturn, dec_delegreturn),
 };
 
 struct rpc_version		nfs_version4 = {

include/linux/nfs4.h

@@ -380,6 +380,7 @@ enum {
 	NFSPROC4_CLNT_READLINK,
 	NFSPROC4_CLNT_READDIR,
 	NFSPROC4_CLNT_SERVER_CAPS,
+	NFSPROC4_CLNT_DELEGRETURN,
 };
 
 #endif

include/linux/nfs_fs.h

@@ -28,6 +28,7 @@
 #include <linux/nfs3.h>
 #include <linux/nfs4.h>
 #include <linux/nfs_xdr.h>
+#include <linux/rwsem.h>
 #include <linux/workqueue.h>
 
 /*
@@ -97,6 +98,11 @@ struct nfs_open_context {
 	wait_queue_head_t waitq;
 };
 
+/*
+ * NFSv4 delegation
+ */
+struct nfs_delegation;
+
 /*
  * nfs fs inode data in memory
  */
@@ -178,6 +184,8 @@ struct nfs_inode {
 #ifdef CONFIG_NFS_V4
         /* NFSv4 state */
 	struct list_head	open_states;
+	struct nfs_delegation	*delegation;
+	struct rw_semaphore	rwsem;
 #endif /* CONFIG_NFS_V4*/
 
 	struct inode		vfs_inode;
@@ -545,6 +553,7 @@ struct nfs4_client {
 	 */
 	struct rw_semaphore	cl_sem;
 
+	struct list_head	cl_delegations;
 	struct list_head	cl_state_owners;
 	struct list_head	cl_unused;
 	int			cl_nunused;
@@ -592,6 +601,7 @@ struct nfs4_state_owner {
 
 	struct rpc_cred	     *so_cred;	 /* Associated cred */
 	struct list_head     so_states;
+	struct list_head     so_delegations;
 };
 
 /*

include/linux/nfs_xdr.h

@@ -122,6 +122,10 @@ struct nfs_openres {
 	__u32                   rflags;
 	struct nfs_fattr *      f_attr;
 	const struct nfs_server *server;
+	int			delegation_type;
+	nfs4_stateid		delegation;
+	__u32			do_recall;
+	__u64			maxsize;
 };
 
 /*
@@ -224,6 +228,11 @@ struct nfs_lockres {
 	const struct nfs_server *	server;
 };
 
+struct nfs4_delegreturnargs {
+	const struct nfs_fh *fhandle;
+	const nfs4_stateid *stateid;
+};
+
 /*
  * Arguments to the read call.
  */