Commit c70422f7 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'nfsd-4.12' of git://linux-nfs.org/~bfields/linux

Pull nfsd updates from Bruce Fields:
 "Another RDMA update from Chuck Lever, and a bunch of miscellaneous
  bugfixes"

* tag 'nfsd-4.12' of git://linux-nfs.org/~bfields/linux: (26 commits)
  nfsd: Fix up the "supattr_exclcreat" attributes
  nfsd: encoders mustn't use unitialized values in error cases
  nfsd: fix undefined behavior in nfsd4_layout_verify
  lockd: fix lockd shutdown race
  NFSv4: Fix callback server shutdown
  SUNRPC: Refactor svc_set_num_threads()
  NFSv4.x/callback: Create the callback service through svc_create_pooled
  lockd: remove redundant check on block
  svcrdma: Clean out old XDR encoders
  svcrdma: Remove the req_map cache
  svcrdma: Remove unused RDMA Write completion handler
  svcrdma: Reduce size of sge array in struct svc_rdma_op_ctxt
  svcrdma: Clean up RPC-over-RDMA backchannel reply processing
  svcrdma: Report Write/Reply chunk overruns
  svcrdma: Clean up RDMA_ERROR path
  svcrdma: Use rdma_rw API in RPC reply path
  svcrdma: Introduce local rdma_rw API helpers
  svcrdma: Clean up svc_rdma_get_inv_rkey()
  svcrdma: Add helper to save pages under I/O
  svcrdma: Eliminate RPCRDMA_SQ_DEPTH_MULT
  ...
parents 73ccb023 b26b78cb
...@@ -132,6 +132,8 @@ lockd(void *vrqstp) ...@@ -132,6 +132,8 @@ lockd(void *vrqstp)
{ {
int err = 0; int err = 0;
struct svc_rqst *rqstp = vrqstp; struct svc_rqst *rqstp = vrqstp;
struct net *net = &init_net;
struct lockd_net *ln = net_generic(net, lockd_net_id);
/* try_to_freeze() is called from svc_recv() */ /* try_to_freeze() is called from svc_recv() */
set_freezable(); set_freezable();
...@@ -176,6 +178,8 @@ lockd(void *vrqstp) ...@@ -176,6 +178,8 @@ lockd(void *vrqstp)
if (nlmsvc_ops) if (nlmsvc_ops)
nlmsvc_invalidate_all(); nlmsvc_invalidate_all();
nlm_shutdown_hosts(); nlm_shutdown_hosts();
cancel_delayed_work_sync(&ln->grace_period_end);
locks_end_grace(&ln->lockd_manager);
return 0; return 0;
} }
...@@ -270,8 +274,6 @@ static void lockd_down_net(struct svc_serv *serv, struct net *net) ...@@ -270,8 +274,6 @@ static void lockd_down_net(struct svc_serv *serv, struct net *net)
if (ln->nlmsvc_users) { if (ln->nlmsvc_users) {
if (--ln->nlmsvc_users == 0) { if (--ln->nlmsvc_users == 0) {
nlm_shutdown_hosts_net(net); nlm_shutdown_hosts_net(net);
cancel_delayed_work_sync(&ln->grace_period_end);
locks_end_grace(&ln->lockd_manager);
svc_shutdown_net(serv, net); svc_shutdown_net(serv, net);
dprintk("lockd_down_net: per-net data destroyed; net=%p\n", net); dprintk("lockd_down_net: per-net data destroyed; net=%p\n", net);
} }
......
...@@ -870,16 +870,16 @@ nlmsvc_grant_reply(struct nlm_cookie *cookie, __be32 status) ...@@ -870,16 +870,16 @@ nlmsvc_grant_reply(struct nlm_cookie *cookie, __be32 status)
if (!(block = nlmsvc_find_block(cookie))) if (!(block = nlmsvc_find_block(cookie)))
return; return;
if (block) {
if (status == nlm_lck_denied_grace_period) { if (status == nlm_lck_denied_grace_period) {
/* Try again in a couple of seconds */ /* Try again in a couple of seconds */
nlmsvc_insert_block(block, 10 * HZ); nlmsvc_insert_block(block, 10 * HZ);
} else { } else {
/* Lock is now held by client, or has been rejected. /*
* In both cases, the block should be removed. */ * Lock is now held by client, or has been rejected.
* In both cases, the block should be removed.
*/
nlmsvc_unlink_block(block); nlmsvc_unlink_block(block);
} }
}
nlmsvc_release_block(block); nlmsvc_release_block(block);
} }
......
...@@ -76,7 +76,10 @@ nfs4_callback_svc(void *vrqstp) ...@@ -76,7 +76,10 @@ nfs4_callback_svc(void *vrqstp)
set_freezable(); set_freezable();
while (!kthread_should_stop()) { while (!kthread_freezable_should_stop(NULL)) {
if (signal_pending(current))
flush_signals(current);
/* /*
* Listen for a request on the socket * Listen for a request on the socket
*/ */
...@@ -85,6 +88,8 @@ nfs4_callback_svc(void *vrqstp) ...@@ -85,6 +88,8 @@ nfs4_callback_svc(void *vrqstp)
continue; continue;
svc_process(rqstp); svc_process(rqstp);
} }
svc_exit_thread(rqstp);
module_put_and_exit(0);
return 0; return 0;
} }
...@@ -103,9 +108,10 @@ nfs41_callback_svc(void *vrqstp) ...@@ -103,9 +108,10 @@ nfs41_callback_svc(void *vrqstp)
set_freezable(); set_freezable();
while (!kthread_should_stop()) { while (!kthread_freezable_should_stop(NULL)) {
if (try_to_freeze())
continue; if (signal_pending(current))
flush_signals(current);
prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_INTERRUPTIBLE); prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_INTERRUPTIBLE);
spin_lock_bh(&serv->sv_cb_lock); spin_lock_bh(&serv->sv_cb_lock);
...@@ -121,11 +127,13 @@ nfs41_callback_svc(void *vrqstp) ...@@ -121,11 +127,13 @@ nfs41_callback_svc(void *vrqstp)
error); error);
} else { } else {
spin_unlock_bh(&serv->sv_cb_lock); spin_unlock_bh(&serv->sv_cb_lock);
if (!kthread_should_stop())
schedule(); schedule();
finish_wait(&serv->sv_cb_waitq, &wq); finish_wait(&serv->sv_cb_waitq, &wq);
} }
flush_signals(current);
} }
svc_exit_thread(rqstp);
module_put_and_exit(0);
return 0; return 0;
} }
...@@ -221,14 +229,14 @@ static int nfs_callback_up_net(int minorversion, struct svc_serv *serv, ...@@ -221,14 +229,14 @@ static int nfs_callback_up_net(int minorversion, struct svc_serv *serv,
static struct svc_serv_ops nfs40_cb_sv_ops = { static struct svc_serv_ops nfs40_cb_sv_ops = {
.svo_function = nfs4_callback_svc, .svo_function = nfs4_callback_svc,
.svo_enqueue_xprt = svc_xprt_do_enqueue, .svo_enqueue_xprt = svc_xprt_do_enqueue,
.svo_setup = svc_set_num_threads, .svo_setup = svc_set_num_threads_sync,
.svo_module = THIS_MODULE, .svo_module = THIS_MODULE,
}; };
#if defined(CONFIG_NFS_V4_1) #if defined(CONFIG_NFS_V4_1)
static struct svc_serv_ops nfs41_cb_sv_ops = { static struct svc_serv_ops nfs41_cb_sv_ops = {
.svo_function = nfs41_callback_svc, .svo_function = nfs41_callback_svc,
.svo_enqueue_xprt = svc_xprt_do_enqueue, .svo_enqueue_xprt = svc_xprt_do_enqueue,
.svo_setup = svc_set_num_threads, .svo_setup = svc_set_num_threads_sync,
.svo_module = THIS_MODULE, .svo_module = THIS_MODULE,
}; };
...@@ -280,7 +288,7 @@ static struct svc_serv *nfs_callback_create_svc(int minorversion) ...@@ -280,7 +288,7 @@ static struct svc_serv *nfs_callback_create_svc(int minorversion)
printk(KERN_WARNING "nfs_callback_create_svc: no kthread, %d users??\n", printk(KERN_WARNING "nfs_callback_create_svc: no kthread, %d users??\n",
cb_info->users); cb_info->users);
serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, sv_ops); serv = svc_create_pooled(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, sv_ops);
if (!serv) { if (!serv) {
printk(KERN_ERR "nfs_callback_create_svc: create service failed\n"); printk(KERN_ERR "nfs_callback_create_svc: create service failed\n");
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
......
...@@ -334,8 +334,11 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -334,8 +334,11 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
if (!p) if (!p)
return 0; return 0;
p = xdr_decode_hyper(p, &args->offset); p = xdr_decode_hyper(p, &args->offset);
args->count = ntohl(*p++); args->count = ntohl(*p++);
if (!xdr_argsize_check(rqstp, p))
return 0;
len = min(args->count, max_blocksize); len = min(args->count, max_blocksize);
/* set up the kvec */ /* set up the kvec */
...@@ -349,7 +352,7 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -349,7 +352,7 @@ nfs3svc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
v++; v++;
} }
args->vlen = v; args->vlen = v;
return xdr_argsize_check(rqstp, p); return 1;
} }
int int
...@@ -541,9 +544,11 @@ nfs3svc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -541,9 +544,11 @@ nfs3svc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p,
p = decode_fh(p, &args->fh); p = decode_fh(p, &args->fh);
if (!p) if (!p)
return 0; return 0;
if (!xdr_argsize_check(rqstp, p))
return 0;
args->buffer = page_address(*(rqstp->rq_next_page++)); args->buffer = page_address(*(rqstp->rq_next_page++));
return xdr_argsize_check(rqstp, p); return 1;
} }
int int
...@@ -569,10 +574,14 @@ nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -569,10 +574,14 @@ nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p,
args->verf = p; p += 2; args->verf = p; p += 2;
args->dircount = ~0; args->dircount = ~0;
args->count = ntohl(*p++); args->count = ntohl(*p++);
if (!xdr_argsize_check(rqstp, p))
return 0;
args->count = min_t(u32, args->count, PAGE_SIZE); args->count = min_t(u32, args->count, PAGE_SIZE);
args->buffer = page_address(*(rqstp->rq_next_page++)); args->buffer = page_address(*(rqstp->rq_next_page++));
return xdr_argsize_check(rqstp, p); return 1;
} }
int int
...@@ -590,6 +599,9 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -590,6 +599,9 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p,
args->dircount = ntohl(*p++); args->dircount = ntohl(*p++);
args->count = ntohl(*p++); args->count = ntohl(*p++);
if (!xdr_argsize_check(rqstp, p))
return 0;
len = args->count = min(args->count, max_blocksize); len = args->count = min(args->count, max_blocksize);
while (len > 0) { while (len > 0) {
struct page *p = *(rqstp->rq_next_page++); struct page *p = *(rqstp->rq_next_page++);
...@@ -597,8 +609,7 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -597,8 +609,7 @@ nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, __be32 *p,
args->buffer = page_address(p); args->buffer = page_address(p);
len -= PAGE_SIZE; len -= PAGE_SIZE;
} }
return 1;
return xdr_argsize_check(rqstp, p);
} }
int int
......
...@@ -1259,7 +1259,8 @@ nfsd4_layout_verify(struct svc_export *exp, unsigned int layout_type) ...@@ -1259,7 +1259,8 @@ nfsd4_layout_verify(struct svc_export *exp, unsigned int layout_type)
return NULL; return NULL;
} }
if (!(exp->ex_layout_types & (1 << layout_type))) { if (layout_type >= LAYOUT_TYPE_MAX ||
!(exp->ex_layout_types & (1 << layout_type))) {
dprintk("%s: layout type %d not supported\n", dprintk("%s: layout type %d not supported\n",
__func__, layout_type); __func__, layout_type);
return NULL; return NULL;
......
...@@ -1912,28 +1912,15 @@ static void copy_clid(struct nfs4_client *target, struct nfs4_client *source) ...@@ -1912,28 +1912,15 @@ static void copy_clid(struct nfs4_client *target, struct nfs4_client *source)
target->cl_clientid.cl_id = source->cl_clientid.cl_id; target->cl_clientid.cl_id = source->cl_clientid.cl_id;
} }
int strdup_if_nonnull(char **target, char *source)
{
if (source) {
*target = kstrdup(source, GFP_KERNEL);
if (!*target)
return -ENOMEM;
} else
*target = NULL;
return 0;
}
static int copy_cred(struct svc_cred *target, struct svc_cred *source) static int copy_cred(struct svc_cred *target, struct svc_cred *source)
{ {
int ret; target->cr_principal = kstrdup(source->cr_principal, GFP_KERNEL);
target->cr_raw_principal = kstrdup(source->cr_raw_principal,
GFP_KERNEL);
if ((source->cr_principal && ! target->cr_principal) ||
(source->cr_raw_principal && ! target->cr_raw_principal))
return -ENOMEM;
ret = strdup_if_nonnull(&target->cr_principal, source->cr_principal);
if (ret)
return ret;
ret = strdup_if_nonnull(&target->cr_raw_principal,
source->cr_raw_principal);
if (ret)
return ret;
target->cr_flavor = source->cr_flavor; target->cr_flavor = source->cr_flavor;
target->cr_uid = source->cr_uid; target->cr_uid = source->cr_uid;
target->cr_gid = source->cr_gid; target->cr_gid = source->cr_gid;
......
...@@ -2831,9 +2831,14 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp, ...@@ -2831,9 +2831,14 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
} }
#endif /* CONFIG_NFSD_PNFS */ #endif /* CONFIG_NFSD_PNFS */
if (bmval2 & FATTR4_WORD2_SUPPATTR_EXCLCREAT) { if (bmval2 & FATTR4_WORD2_SUPPATTR_EXCLCREAT) {
status = nfsd4_encode_bitmap(xdr, NFSD_SUPPATTR_EXCLCREAT_WORD0, u32 supp[3];
NFSD_SUPPATTR_EXCLCREAT_WORD1,
NFSD_SUPPATTR_EXCLCREAT_WORD2); memcpy(supp, nfsd_suppattrs[minorversion], sizeof(supp));
supp[0] &= NFSD_SUPPATTR_EXCLCREAT_WORD0;
supp[1] &= NFSD_SUPPATTR_EXCLCREAT_WORD1;
supp[2] &= NFSD_SUPPATTR_EXCLCREAT_WORD2;
status = nfsd4_encode_bitmap(xdr, supp[0], supp[1], supp[2]);
if (status) if (status)
goto out; goto out;
} }
...@@ -4119,8 +4124,7 @@ nfsd4_encode_getdeviceinfo(struct nfsd4_compoundres *resp, __be32 nfserr, ...@@ -4119,8 +4124,7 @@ nfsd4_encode_getdeviceinfo(struct nfsd4_compoundres *resp, __be32 nfserr,
struct nfsd4_getdeviceinfo *gdev) struct nfsd4_getdeviceinfo *gdev)
{ {
struct xdr_stream *xdr = &resp->xdr; struct xdr_stream *xdr = &resp->xdr;
const struct nfsd4_layout_ops *ops = const struct nfsd4_layout_ops *ops;
nfsd4_layout_ops[gdev->gd_layout_type];
u32 starting_len = xdr->buf->len, needed_len; u32 starting_len = xdr->buf->len, needed_len;
__be32 *p; __be32 *p;
...@@ -4137,6 +4141,7 @@ nfsd4_encode_getdeviceinfo(struct nfsd4_compoundres *resp, __be32 nfserr, ...@@ -4137,6 +4141,7 @@ nfsd4_encode_getdeviceinfo(struct nfsd4_compoundres *resp, __be32 nfserr,
/* If maxcount is 0 then just update notifications */ /* If maxcount is 0 then just update notifications */
if (gdev->gd_maxcount != 0) { if (gdev->gd_maxcount != 0) {
ops = nfsd4_layout_ops[gdev->gd_layout_type];
nfserr = ops->encode_getdeviceinfo(xdr, gdev); nfserr = ops->encode_getdeviceinfo(xdr, gdev);
if (nfserr) { if (nfserr) {
/* /*
...@@ -4189,8 +4194,7 @@ nfsd4_encode_layoutget(struct nfsd4_compoundres *resp, __be32 nfserr, ...@@ -4189,8 +4194,7 @@ nfsd4_encode_layoutget(struct nfsd4_compoundres *resp, __be32 nfserr,
struct nfsd4_layoutget *lgp) struct nfsd4_layoutget *lgp)
{ {
struct xdr_stream *xdr = &resp->xdr; struct xdr_stream *xdr = &resp->xdr;
const struct nfsd4_layout_ops *ops = const struct nfsd4_layout_ops *ops;
nfsd4_layout_ops[lgp->lg_layout_type];
__be32 *p; __be32 *p;
dprintk("%s: err %d\n", __func__, nfserr); dprintk("%s: err %d\n", __func__, nfserr);
...@@ -4213,6 +4217,7 @@ nfsd4_encode_layoutget(struct nfsd4_compoundres *resp, __be32 nfserr, ...@@ -4213,6 +4217,7 @@ nfsd4_encode_layoutget(struct nfsd4_compoundres *resp, __be32 nfserr,
*p++ = cpu_to_be32(lgp->lg_seg.iomode); *p++ = cpu_to_be32(lgp->lg_seg.iomode);
*p++ = cpu_to_be32(lgp->lg_layout_type); *p++ = cpu_to_be32(lgp->lg_layout_type);
ops = nfsd4_layout_ops[lgp->lg_layout_type];
nfserr = ops->encode_layoutget(xdr, lgp); nfserr = ops->encode_layoutget(xdr, lgp);
out: out:
kfree(lgp->lg_content); kfree(lgp->lg_content);
......
...@@ -257,6 +257,9 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -257,6 +257,9 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
len = args->count = ntohl(*p++); len = args->count = ntohl(*p++);
p++; /* totalcount - unused */ p++; /* totalcount - unused */
if (!xdr_argsize_check(rqstp, p))
return 0;
len = min_t(unsigned int, len, NFSSVC_MAXBLKSIZE_V2); len = min_t(unsigned int, len, NFSSVC_MAXBLKSIZE_V2);
/* set up somewhere to store response. /* set up somewhere to store response.
...@@ -272,7 +275,7 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -272,7 +275,7 @@ nfssvc_decode_readargs(struct svc_rqst *rqstp, __be32 *p,
v++; v++;
} }
args->vlen = v; args->vlen = v;
return xdr_argsize_check(rqstp, p); return 1;
} }
int int
...@@ -362,9 +365,11 @@ nfssvc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p, struct nfsd_readli ...@@ -362,9 +365,11 @@ nfssvc_decode_readlinkargs(struct svc_rqst *rqstp, __be32 *p, struct nfsd_readli
p = decode_fh(p, &args->fh); p = decode_fh(p, &args->fh);
if (!p) if (!p)
return 0; return 0;
if (!xdr_argsize_check(rqstp, p))
return 0;
args->buffer = page_address(*(rqstp->rq_next_page++)); args->buffer = page_address(*(rqstp->rq_next_page++));
return xdr_argsize_check(rqstp, p); return 1;
} }
int int
...@@ -402,9 +407,11 @@ nfssvc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p, ...@@ -402,9 +407,11 @@ nfssvc_decode_readdirargs(struct svc_rqst *rqstp, __be32 *p,
args->cookie = ntohl(*p++); args->cookie = ntohl(*p++);
args->count = ntohl(*p++); args->count = ntohl(*p++);
args->count = min_t(u32, args->count, PAGE_SIZE); args->count = min_t(u32, args->count, PAGE_SIZE);
if (!xdr_argsize_check(rqstp, p))
return 0;
args->buffer = page_address(*(rqstp->rq_next_page++)); args->buffer = page_address(*(rqstp->rq_next_page++));
return xdr_argsize_check(rqstp, p); return 1;
} }
/* /*
......
...@@ -94,6 +94,12 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, ...@@ -94,6 +94,12 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
err = follow_down(&path); err = follow_down(&path);
if (err < 0) if (err < 0)
goto out; goto out;
if (path.mnt == exp->ex_path.mnt && path.dentry == dentry &&
nfsd_mountpoint(dentry, exp) == 2) {
/* This is only a mountpoint in some other namespace */
path_put(&path);
goto out;
}
exp2 = rqst_exp_get_by_name(rqstp, &path); exp2 = rqst_exp_get_by_name(rqstp, &path);
if (IS_ERR(exp2)) { if (IS_ERR(exp2)) {
...@@ -167,16 +173,26 @@ static int nfsd_lookup_parent(struct svc_rqst *rqstp, struct dentry *dparent, st ...@@ -167,16 +173,26 @@ static int nfsd_lookup_parent(struct svc_rqst *rqstp, struct dentry *dparent, st
/* /*
* For nfsd purposes, we treat V4ROOT exports as though there was an * For nfsd purposes, we treat V4ROOT exports as though there was an
* export at *every* directory. * export at *every* directory.
* We return:
* '1' if this dentry *must* be an export point,
* '2' if it might be, if there is really a mount here, and
* '0' if there is no chance of an export point here.
*/ */
int nfsd_mountpoint(struct dentry *dentry, struct svc_export *exp) int nfsd_mountpoint(struct dentry *dentry, struct svc_export *exp)
{ {
if (d_mountpoint(dentry)) if (!d_inode(dentry))
return 0;
if (exp->ex_flags & NFSEXP_V4ROOT)
return 1; return 1;
if (nfsd4_is_junction(dentry)) if (nfsd4_is_junction(dentry))
return 1; return 1;
if (!(exp->ex_flags & NFSEXP_V4ROOT)) if (d_mountpoint(dentry))
/*
* Might only be a mountpoint in a different namespace,
* but we need to check.
*/
return 2;
return 0; return 0;
return d_inode(dentry) != NULL;
} }
__be32 __be32
......
...@@ -143,6 +143,9 @@ enum rpcrdma_proc { ...@@ -143,6 +143,9 @@ enum rpcrdma_proc {
#define rdma_done cpu_to_be32(RDMA_DONE) #define rdma_done cpu_to_be32(RDMA_DONE)
#define rdma_error cpu_to_be32(RDMA_ERROR) #define rdma_error cpu_to_be32(RDMA_ERROR)
#define err_vers cpu_to_be32(ERR_VERS)
#define err_chunk cpu_to_be32(ERR_CHUNK)
/* /*
* Private extension to RPC-over-RDMA Version One. * Private extension to RPC-over-RDMA Version One.
* Message passed during RDMA-CM connection set-up. * Message passed during RDMA-CM connection set-up.
......
...@@ -336,8 +336,7 @@ xdr_argsize_check(struct svc_rqst *rqstp, __be32 *p) ...@@ -336,8 +336,7 @@ xdr_argsize_check(struct svc_rqst *rqstp, __be32 *p)
{ {
char *cp = (char *)p; char *cp = (char *)p;
struct kvec *vec = &rqstp->rq_arg.head[0]; struct kvec *vec = &rqstp->rq_arg.head[0];
return cp >= (char*)vec->iov_base return cp == (char *)vec->iov_base + vec->iov_len;
&& cp <= (char*)vec->iov_base + vec->iov_len;
} }
static inline int static inline int
...@@ -474,6 +473,7 @@ void svc_pool_map_put(void); ...@@ -474,6 +473,7 @@ void svc_pool_map_put(void);
struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int,
struct svc_serv_ops *); struct svc_serv_ops *);
int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int);
int svc_set_num_threads_sync(struct svc_serv *, struct svc_pool *, int);
int svc_pool_stats_open(struct svc_serv *serv, struct file *file); int svc_pool_stats_open(struct svc_serv *serv, struct file *file);
void svc_destroy(struct svc_serv *); void svc_destroy(struct svc_serv *);
void svc_shutdown_net(struct svc_serv *, struct net *); void svc_shutdown_net(struct svc_serv *, struct net *);
......
...@@ -48,6 +48,12 @@ ...@@ -48,6 +48,12 @@
#include <rdma/rdma_cm.h> #include <rdma/rdma_cm.h>
#define SVCRDMA_DEBUG #define SVCRDMA_DEBUG
/* Default and maximum inline threshold sizes */
enum {
RPCRDMA_DEF_INLINE_THRESH = 4096,
RPCRDMA_MAX_INLINE_THRESH = 65536
};
/* RPC/RDMA parameters and stats */ /* RPC/RDMA parameters and stats */
extern unsigned int svcrdma_ord; extern unsigned int svcrdma_ord;
extern unsigned int svcrdma_max_requests; extern unsigned int svcrdma_max_requests;
...@@ -85,27 +91,11 @@ struct svc_rdma_op_ctxt { ...@@ -85,27 +91,11 @@ struct svc_rdma_op_ctxt {
enum dma_data_direction direction; enum dma_data_direction direction;
int count; int count;
unsigned int mapped_sges; unsigned int mapped_sges;
struct ib_sge sge[RPCSVC_MAXPAGES]; struct ib_send_wr send_wr;
struct ib_sge sge[1 + RPCRDMA_MAX_INLINE_THRESH / PAGE_SIZE];
struct page *pages[RPCSVC_MAXPAGES]; struct page *pages[RPCSVC_MAXPAGES];
}; };
/*
* NFS_ requests are mapped on the client side by the chunk lists in
* the RPCRDMA header. During the fetching of the RPC from the client
* and the writing of the reply to the client, the memory in the
* client and the memory in the server must be mapped as contiguous
* vaddr/len for access by the hardware. These data strucures keep
* these mappings.
*
* For an RDMA_WRITE, the 'sge' maps the RPC REPLY. For RDMA_READ, the
* 'sge' in the svc_rdma_req_map maps the server side RPC reply and the
* 'ch' field maps the read-list of the RPCRDMA header to the 'sge'
* mapping of the reply.
*/
struct svc_rdma_chunk_sge {
int start; /* sge no for this chunk */
int count; /* sge count for this chunk */
};
struct svc_rdma_fastreg_mr { struct svc_rdma_fastreg_mr {
struct ib_mr *mr; struct ib_mr *mr;
struct scatterlist *sg; struct scatterlist *sg;
...@@ -114,15 +104,7 @@ struct svc_rdma_fastreg_mr { ...@@ -114,15 +104,7 @@ struct svc_rdma_fastreg_mr {
enum dma_data_direction direction; enum dma_data_direction direction;
struct list_head frmr_list; struct list_head frmr_list;
}; };
struct svc_rdma_req_map {
struct list_head free;
unsigned long count;
union {
struct kvec sge[RPCSVC_MAXPAGES];
struct svc_rdma_chunk_sge ch[RPCSVC_MAXPAGES];
unsigned long lkey[RPCSVC_MAXPAGES];
};
};
#define RDMACTXT_F_LAST_CTXT 2 #define RDMACTXT_F_LAST_CTXT 2
#define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */ #define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */
...@@ -144,14 +126,15 @@ struct svcxprt_rdma { ...@@ -144,14 +126,15 @@ struct svcxprt_rdma {
u32 sc_max_requests; /* Max requests */ u32 sc_max_requests; /* Max requests */
u32 sc_max_bc_requests;/* Backward credits */ u32 sc_max_bc_requests;/* Backward credits */
int sc_max_req_size; /* Size of each RQ WR buf */ int sc_max_req_size; /* Size of each RQ WR buf */
u8 sc_port_num;
struct ib_pd *sc_pd; struct ib_pd *sc_pd;
spinlock_t sc_ctxt_lock; spinlock_t sc_ctxt_lock;
struct list_head sc_ctxts; struct list_head sc_ctxts;
int sc_ctxt_used; int sc_ctxt_used;
spinlock_t sc_map_lock; spinlock_t sc_rw_ctxt_lock;
struct list_head sc_maps; struct list_head sc_rw_ctxts;
struct list_head sc_rq_dto_q; struct list_head sc_rq_dto_q;
spinlock_t sc_rq_dto_lock; spinlock_t sc_rq_dto_lock;
...@@ -181,9 +164,7 @@ struct svcxprt_rdma { ...@@ -181,9 +164,7 @@ struct svcxprt_rdma {
/* The default ORD value is based on two outstanding full-size writes with a /* The default ORD value is based on two outstanding full-size writes with a
* page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */ * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */
#define RPCRDMA_ORD (64/4) #define RPCRDMA_ORD (64/4)
#define RPCRDMA_SQ_DEPTH_MULT 8
#define RPCRDMA_MAX_REQUESTS 32 #define RPCRDMA_MAX_REQUESTS 32
#define RPCRDMA_MAX_REQ_SIZE 4096
/* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our /* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our
* current NFSv4.1 implementation supports one backchannel slot. * current NFSv4.1 implementation supports one backchannel slot.
...@@ -201,19 +182,11 @@ static inline void svc_rdma_count_mappings(struct svcxprt_rdma *rdma, ...@@ -201,19 +182,11 @@ static inline void svc_rdma_count_mappings(struct svcxprt_rdma *rdma,
/* svc_rdma_backchannel.c */ /* svc_rdma_backchannel.c */
extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt,
struct rpcrdma_msg *rmsgp, __be32 *rdma_resp,
struct xdr_buf *rcvbuf); struct xdr_buf *rcvbuf);
/* svc_rdma_marshal.c */ /* svc_rdma_marshal.c */
extern int svc_rdma_xdr_decode_req(struct xdr_buf *); extern int svc_rdma_xdr_decode_req(struct xdr_buf *);
extern int svc_rdma_xdr_encode_error(struct svcxprt_rdma *,
struct rpcrdma_msg *,
enum rpcrdma_errcode, __be32 *);
extern void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg *, int);
extern void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array *, int);
extern void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array *, int,
__be32, __be64, u32);
extern unsigned int svc_rdma_xdr_get_reply_hdr_len(__be32 *rdma_resp);
/* svc_rdma_recvfrom.c */ /* svc_rdma_recvfrom.c */
extern int svc_rdma_recvfrom(struct svc_rqst *); extern int svc_rdma_recvfrom(struct svc_rqst *);
...@@ -224,16 +197,25 @@ extern int rdma_read_chunk_frmr(struct svcxprt_rdma *, struct svc_rqst *, ...@@ -224,16 +197,25 @@ extern int rdma_read_chunk_frmr(struct svcxprt_rdma *, struct svc_rqst *,
struct svc_rdma_op_ctxt *, int *, u32 *, struct svc_rdma_op_ctxt *, int *, u32 *,
u32, u32, u64, bool); u32, u32, u64, bool);
/* svc_rdma_rw.c */
extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
__be32 *wr_ch, struct xdr_buf *xdr);
extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
__be32 *rp_ch, bool writelist,
struct xdr_buf *xdr);
/* svc_rdma_sendto.c */ /* svc_rdma_sendto.c */
extern int svc_rdma_map_xdr(struct svcxprt_rdma *, struct xdr_buf *, extern int svc_rdma_map_reply_hdr(struct svcxprt_rdma *rdma,
struct svc_rdma_req_map *, bool); struct svc_rdma_op_ctxt *ctxt,
__be32 *rdma_resp, unsigned int len);
extern int svc_rdma_post_send_wr(struct svcxprt_rdma *rdma,
struct svc_rdma_op_ctxt *ctxt,
int num_sge, u32 inv_rkey);
extern int svc_rdma_sendto(struct svc_rqst *); extern int svc_rdma_sendto(struct svc_rqst *);
extern void svc_rdma_send_error(struct svcxprt_rdma *, struct rpcrdma_msg *,
int);
/* svc_rdma_transport.c */ /* svc_rdma_transport.c */
extern void svc_rdma_wc_send(struct ib_cq *, struct ib_wc *); extern void svc_rdma_wc_send(struct ib_cq *, struct ib_wc *);
extern void svc_rdma_wc_write(struct ib_cq *, struct ib_wc *);
extern void svc_rdma_wc_reg(struct ib_cq *, struct ib_wc *); extern void svc_rdma_wc_reg(struct ib_cq *, struct ib_wc *);
extern void svc_rdma_wc_read(struct ib_cq *, struct ib_wc *); extern void svc_rdma_wc_read(struct ib_cq *, struct ib_wc *);
extern void svc_rdma_wc_inv(struct ib_cq *, struct ib_wc *); extern void svc_rdma_wc_inv(struct ib_cq *, struct ib_wc *);
...@@ -244,9 +226,6 @@ extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *); ...@@ -244,9 +226,6 @@ extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *);
extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *); extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *);
extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int); extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int);
extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt); extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt);
extern struct svc_rdma_req_map *svc_rdma_get_req_map(struct svcxprt_rdma *);
extern void svc_rdma_put_req_map(struct svcxprt_rdma *,
struct svc_rdma_req_map *);
extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *); extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *);
extern void svc_rdma_put_frmr(struct svcxprt_rdma *, extern void svc_rdma_put_frmr(struct svcxprt_rdma *,
struct svc_rdma_fastreg_mr *); struct svc_rdma_fastreg_mr *);
......
...@@ -22,6 +22,8 @@ ...@@ -22,6 +22,8 @@
#ifndef _NFSD_CLD_H #ifndef _NFSD_CLD_H
#define _NFSD_CLD_H #define _NFSD_CLD_H
#include <linux/types.h>
/* latest upcall version available */ /* latest upcall version available */
#define CLD_UPCALL_VERSION 1 #define CLD_UPCALL_VERSION 1
...@@ -37,18 +39,18 @@ enum cld_command { ...@@ -37,18 +39,18 @@ enum cld_command {
/* representation of long-form NFSv4 client ID */ /* representation of long-form NFSv4 client ID */
struct cld_name { struct cld_name {
uint16_t cn_len; /* length of cm_id */ __u16 cn_len; /* length of cm_id */
unsigned char cn_id[NFS4_OPAQUE_LIMIT]; /* client-provided */ unsigned char cn_id[NFS4_OPAQUE_LIMIT]; /* client-provided */
} __attribute__((packed)); } __attribute__((packed));
/* message struct for communication with userspace */ /* message struct for communication with userspace */
struct cld_msg { struct cld_msg {
uint8_t cm_vers; /* upcall version */ __u8 cm_vers; /* upcall version */
uint8_t cm_cmd; /* upcall command */ __u8 cm_cmd; /* upcall command */
int16_t cm_status; /* return code */ __s16 cm_status; /* return code */
uint32_t cm_xid; /* transaction id */ __u32 cm_xid; /* transaction id */
union { union {
int64_t cm_gracetime; /* grace period start time */ __s64 cm_gracetime; /* grace period start time */
struct cld_name cm_name; struct cld_name cm_name;
} __attribute__((packed)) cm_u; } __attribute__((packed)) cm_u;
} __attribute__((packed)); } __attribute__((packed));
......
...@@ -52,6 +52,7 @@ config SUNRPC_XPRT_RDMA ...@@ -52,6 +52,7 @@ config SUNRPC_XPRT_RDMA
tristate "RPC-over-RDMA transport" tristate "RPC-over-RDMA transport"
depends on SUNRPC && INFINIBAND && INFINIBAND_ADDR_TRANS depends on SUNRPC && INFINIBAND && INFINIBAND_ADDR_TRANS
default SUNRPC && INFINIBAND default SUNRPC && INFINIBAND
select SG_POOL
help help
This option allows the NFS client and server to use RDMA This option allows the NFS client and server to use RDMA
transports (InfiniBand, iWARP, or RoCE). transports (InfiniBand, iWARP, or RoCE).
......
...@@ -702,59 +702,32 @@ choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state) ...@@ -702,59 +702,32 @@ choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
return task; return task;
} }
/* /* create new threads */
* Create or destroy enough new threads to make the number static int
* of threads the given number. If `pool' is non-NULL, applies svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
* only to threads in that pool, otherwise round-robins between
* all pools. Caller must ensure that mutual exclusion between this and
* server startup or shutdown.
*
* Destroying threads relies on the service threads filling in
* rqstp->rq_task, which only the nfs ones do. Assumes the serv
* has been created using svc_create_pooled().
*
* Based on code that used to be in nfsd_svc() but tweaked
* to be pool-aware.
*/
int
svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{ {
struct svc_rqst *rqstp; struct svc_rqst *rqstp;
struct task_struct *task; struct task_struct *task;
struct svc_pool *chosen_pool; struct svc_pool *chosen_pool;
int error = 0;
unsigned int state = serv->sv_nrthreads-1; unsigned int state = serv->sv_nrthreads-1;
int node; int node;
if (pool == NULL) { do {
/* The -1 assumes caller has done a svc_get() */
nrservs -= (serv->sv_nrthreads-1);
} else {
spin_lock_bh(&pool->sp_lock);
nrservs -= pool->sp_nrthreads;
spin_unlock_bh(&pool->sp_lock);
}
/* create new threads */
while (nrservs > 0) {
nrservs--; nrservs--;
chosen_pool = choose_pool(serv, pool, &state); chosen_pool = choose_pool(serv, pool, &state);
node = svc_pool_map_get_node(chosen_pool->sp_id); node = svc_pool_map_get_node(chosen_pool->sp_id);
rqstp = svc_prepare_thread(serv, chosen_pool, node); rqstp = svc_prepare_thread(serv, chosen_pool, node);
if (IS_ERR(rqstp)) { if (IS_ERR(rqstp))
error = PTR_ERR(rqstp); return PTR_ERR(rqstp);
break;
}
__module_get(serv->sv_ops->svo_module); __module_get(serv->sv_ops->svo_module);
task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp, task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
node, "%s", serv->sv_name); node, "%s", serv->sv_name);
if (IS_ERR(task)) { if (IS_ERR(task)) {
error = PTR_ERR(task);
module_put(serv->sv_ops->svo_module); module_put(serv->sv_ops->svo_module);
svc_exit_thread(rqstp); svc_exit_thread(rqstp);
break; return PTR_ERR(task);
} }
rqstp->rq_task = task; rqstp->rq_task = task;
...@@ -763,18 +736,103 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs) ...@@ -763,18 +736,103 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
svc_sock_update_bufs(serv); svc_sock_update_bufs(serv);
wake_up_process(task); wake_up_process(task);
} } while (nrservs > 0);
return 0;
}
/* destroy old threads */
static int
svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
struct task_struct *task;
unsigned int state = serv->sv_nrthreads-1;
/* destroy old threads */ /* destroy old threads */
while (nrservs < 0 && do {
(task = choose_victim(serv, pool, &state)) != NULL) { task = choose_victim(serv, pool, &state);
if (task == NULL)
break;
send_sig(SIGINT, task, 1); send_sig(SIGINT, task, 1);
nrservs++; nrservs++;
} while (nrservs < 0);
return 0;
}
/*
* Create or destroy enough new threads to make the number
* of threads the given number. If `pool' is non-NULL, applies
* only to threads in that pool, otherwise round-robins between
* all pools. Caller must ensure that mutual exclusion between this and
* server startup or shutdown.
*
* Destroying threads relies on the service threads filling in
* rqstp->rq_task, which only the nfs ones do. Assumes the serv
* has been created using svc_create_pooled().
*
* Based on code that used to be in nfsd_svc() but tweaked
* to be pool-aware.
*/
int
svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
if (pool == NULL) {
/* The -1 assumes caller has done a svc_get() */
nrservs -= (serv->sv_nrthreads-1);
} else {
spin_lock_bh(&pool->sp_lock);
nrservs -= pool->sp_nrthreads;
spin_unlock_bh(&pool->sp_lock);
} }
return error; if (nrservs > 0)
return svc_start_kthreads(serv, pool, nrservs);
if (nrservs < 0)
return svc_signal_kthreads(serv, pool, nrservs);
return 0;
} }
EXPORT_SYMBOL_GPL(svc_set_num_threads); EXPORT_SYMBOL_GPL(svc_set_num_threads);
/* destroy old threads */
static int
svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
struct task_struct *task;
unsigned int state = serv->sv_nrthreads-1;
/* destroy old threads */
do {
task = choose_victim(serv, pool, &state);
if (task == NULL)
break;
kthread_stop(task);
nrservs++;
} while (nrservs < 0);
return 0;
}
int
svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
{
if (pool == NULL) {
/* The -1 assumes caller has done a svc_get() */
nrservs -= (serv->sv_nrthreads-1);
} else {
spin_lock_bh(&pool->sp_lock);
nrservs -= pool->sp_nrthreads;
spin_unlock_bh(&pool->sp_lock);
}
if (nrservs > 0)
return svc_start_kthreads(serv, pool, nrservs);
if (nrservs < 0)
return svc_stop_kthreads(serv, pool, nrservs);
return 0;
}
EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);
/* /*
* Called from a server thread as it's exiting. Caller must hold the "service * Called from a server thread as it's exiting. Caller must hold the "service
* mutex" for the service. * mutex" for the service.
......
...@@ -4,5 +4,5 @@ rpcrdma-y := transport.o rpc_rdma.o verbs.o \ ...@@ -4,5 +4,5 @@ rpcrdma-y := transport.o rpc_rdma.o verbs.o \
fmr_ops.o frwr_ops.o \ fmr_ops.o frwr_ops.o \
svc_rdma.o svc_rdma_backchannel.o svc_rdma_transport.o \ svc_rdma.o svc_rdma_backchannel.o svc_rdma_transport.o \
svc_rdma_marshal.o svc_rdma_sendto.o svc_rdma_recvfrom.o \ svc_rdma_marshal.o svc_rdma_sendto.o svc_rdma_recvfrom.o \
module.o svc_rdma_rw.o module.o
rpcrdma-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel.o rpcrdma-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel.o
...@@ -58,9 +58,9 @@ unsigned int svcrdma_max_requests = RPCRDMA_MAX_REQUESTS; ...@@ -58,9 +58,9 @@ unsigned int svcrdma_max_requests = RPCRDMA_MAX_REQUESTS;
unsigned int svcrdma_max_bc_requests = RPCRDMA_MAX_BC_REQUESTS; unsigned int svcrdma_max_bc_requests = RPCRDMA_MAX_BC_REQUESTS;
static unsigned int min_max_requests = 4; static unsigned int min_max_requests = 4;
static unsigned int max_max_requests = 16384; static unsigned int max_max_requests = 16384;
unsigned int svcrdma_max_req_size = RPCRDMA_MAX_REQ_SIZE; unsigned int svcrdma_max_req_size = RPCRDMA_DEF_INLINE_THRESH;
static unsigned int min_max_inline = 4096; static unsigned int min_max_inline = RPCRDMA_DEF_INLINE_THRESH;
static unsigned int max_max_inline = 65536; static unsigned int max_max_inline = RPCRDMA_MAX_INLINE_THRESH;
atomic_t rdma_stat_recv; atomic_t rdma_stat_recv;
atomic_t rdma_stat_read; atomic_t rdma_stat_read;
...@@ -247,8 +247,6 @@ int svc_rdma_init(void) ...@@ -247,8 +247,6 @@ int svc_rdma_init(void)
dprintk("SVCRDMA Module Init, register RPC RDMA transport\n"); dprintk("SVCRDMA Module Init, register RPC RDMA transport\n");
dprintk("\tsvcrdma_ord : %d\n", svcrdma_ord); dprintk("\tsvcrdma_ord : %d\n", svcrdma_ord);
dprintk("\tmax_requests : %u\n", svcrdma_max_requests); dprintk("\tmax_requests : %u\n", svcrdma_max_requests);
dprintk("\tsq_depth : %u\n",
svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
dprintk("\tmax_bc_requests : %u\n", svcrdma_max_bc_requests); dprintk("\tmax_bc_requests : %u\n", svcrdma_max_bc_requests);
dprintk("\tmax_inline : %d\n", svcrdma_max_req_size); dprintk("\tmax_inline : %d\n", svcrdma_max_req_size);
......
...@@ -12,7 +12,17 @@ ...@@ -12,7 +12,17 @@
#undef SVCRDMA_BACKCHANNEL_DEBUG #undef SVCRDMA_BACKCHANNEL_DEBUG
int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp, /**
* svc_rdma_handle_bc_reply - Process incoming backchannel reply
* @xprt: controlling backchannel transport
* @rdma_resp: pointer to incoming transport header
* @rcvbuf: XDR buffer into which to decode the reply
*
* Returns:
* %0 if @rcvbuf is filled in, xprt_complete_rqst called,
* %-EAGAIN if server should call ->recvfrom again.
*/
int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp,
struct xdr_buf *rcvbuf) struct xdr_buf *rcvbuf)
{ {
struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
...@@ -27,13 +37,13 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp, ...@@ -27,13 +37,13 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp,
p = (__be32 *)src->iov_base; p = (__be32 *)src->iov_base;
len = src->iov_len; len = src->iov_len;
xid = rmsgp->rm_xid; xid = *rdma_resp;
#ifdef SVCRDMA_BACKCHANNEL_DEBUG #ifdef SVCRDMA_BACKCHANNEL_DEBUG
pr_info("%s: xid=%08x, length=%zu\n", pr_info("%s: xid=%08x, length=%zu\n",
__func__, be32_to_cpu(xid), len); __func__, be32_to_cpu(xid), len);
pr_info("%s: RPC/RDMA: %*ph\n", pr_info("%s: RPC/RDMA: %*ph\n",
__func__, (int)RPCRDMA_HDRLEN_MIN, rmsgp); __func__, (int)RPCRDMA_HDRLEN_MIN, rdma_resp);
pr_info("%s: RPC: %*ph\n", pr_info("%s: RPC: %*ph\n",
__func__, (int)len, p); __func__, (int)len, p);
#endif #endif
...@@ -53,7 +63,7 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp, ...@@ -53,7 +63,7 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp,
goto out_unlock; goto out_unlock;
memcpy(dst->iov_base, p, len); memcpy(dst->iov_base, p, len);
credits = be32_to_cpu(rmsgp->rm_credit); credits = be32_to_cpup(rdma_resp + 2);
if (credits == 0) if (credits == 0)
credits = 1; /* don't deadlock */ credits = 1; /* don't deadlock */
else if (credits > r_xprt->rx_buf.rb_bc_max_requests) else if (credits > r_xprt->rx_buf.rb_bc_max_requests)
...@@ -90,9 +100,9 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp, ...@@ -90,9 +100,9 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp,
* Caller holds the connection's mutex and has already marshaled * Caller holds the connection's mutex and has already marshaled
* the RPC/RDMA request. * the RPC/RDMA request.
* *
* This is similar to svc_rdma_reply, but takes an rpc_rqst * This is similar to svc_rdma_send_reply_msg, but takes a struct
* instead, does not support chunks, and avoids blocking memory * rpc_rqst instead, does not support chunks, and avoids blocking
* allocation. * memory allocation.
* *
* XXX: There is still an opportunity to block in svc_rdma_send() * XXX: There is still an opportunity to block in svc_rdma_send()
* if there are no SQ entries to post the Send. This may occur if * if there are no SQ entries to post the Send. This may occur if
...@@ -101,59 +111,36 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp, ...@@ -101,59 +111,36 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, struct rpcrdma_msg *rmsgp,
static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma, static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma,
struct rpc_rqst *rqst) struct rpc_rqst *rqst)
{ {
struct xdr_buf *sndbuf = &rqst->rq_snd_buf;
struct svc_rdma_op_ctxt *ctxt; struct svc_rdma_op_ctxt *ctxt;
struct svc_rdma_req_map *vec;
struct ib_send_wr send_wr;
int ret; int ret;
vec = svc_rdma_get_req_map(rdma); ctxt = svc_rdma_get_context(rdma);
ret = svc_rdma_map_xdr(rdma, sndbuf, vec, false);
if (ret) /* rpcrdma_bc_send_request builds the transport header and
* the backchannel RPC message in the same buffer. Thus only
* one SGE is needed to send both.
*/
ret = svc_rdma_map_reply_hdr(rdma, ctxt, rqst->rq_buffer,
rqst->rq_snd_buf.len);
if (ret < 0)
goto out_err; goto out_err;
ret = svc_rdma_repost_recv(rdma, GFP_NOIO); ret = svc_rdma_repost_recv(rdma, GFP_NOIO);
if (ret) if (ret)
goto out_err; goto out_err;
ctxt = svc_rdma_get_context(rdma); ret = svc_rdma_post_send_wr(rdma, ctxt, 1, 0);
ctxt->pages[0] = virt_to_page(rqst->rq_buffer); if (ret)
ctxt->count = 1;
ctxt->direction = DMA_TO_DEVICE;
ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[0].length = sndbuf->len;
ctxt->sge[0].addr =
ib_dma_map_page(rdma->sc_cm_id->device, ctxt->pages[0], 0,
sndbuf->len, DMA_TO_DEVICE);
if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) {
ret = -EIO;
goto out_unmap;
}
svc_rdma_count_mappings(rdma, ctxt);
memset(&send_wr, 0, sizeof(send_wr));
ctxt->cqe.done = svc_rdma_wc_send;
send_wr.wr_cqe = &ctxt->cqe;
send_wr.sg_list = ctxt->sge;
send_wr.num_sge = 1;
send_wr.opcode = IB_WR_SEND;
send_wr.send_flags = IB_SEND_SIGNALED;
ret = svc_rdma_send(rdma, &send_wr);
if (ret) {
ret = -EIO;
goto out_unmap; goto out_unmap;
}
out_err: out_err:
svc_rdma_put_req_map(rdma, vec);
dprintk("svcrdma: %s returns %d\n", __func__, ret); dprintk("svcrdma: %s returns %d\n", __func__, ret);
return ret; return ret;
out_unmap: out_unmap:
svc_rdma_unmap_dma(ctxt); svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1); svc_rdma_put_context(ctxt, 1);
ret = -EIO;
goto out_err; goto out_err;
} }
......
...@@ -166,92 +166,3 @@ int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg) ...@@ -166,92 +166,3 @@ int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg)
dprintk("svcrdma: failed to parse transport header\n"); dprintk("svcrdma: failed to parse transport header\n");
return -EINVAL; return -EINVAL;
} }
int svc_rdma_xdr_encode_error(struct svcxprt_rdma *xprt,
struct rpcrdma_msg *rmsgp,
enum rpcrdma_errcode err, __be32 *va)
{
__be32 *startp = va;
*va++ = rmsgp->rm_xid;
*va++ = rmsgp->rm_vers;
*va++ = xprt->sc_fc_credits;
*va++ = rdma_error;
*va++ = cpu_to_be32(err);
if (err == ERR_VERS) {
*va++ = rpcrdma_version;
*va++ = rpcrdma_version;
}
return (int)((unsigned long)va - (unsigned long)startp);
}
/**
* svc_rdma_xdr_get_reply_hdr_length - Get length of Reply transport header
* @rdma_resp: buffer containing Reply transport header
*
* Returns length of transport header, in bytes.
*/
unsigned int svc_rdma_xdr_get_reply_hdr_len(__be32 *rdma_resp)
{
unsigned int nsegs;
__be32 *p;
p = rdma_resp;
/* RPC-over-RDMA V1 replies never have a Read list. */
p += rpcrdma_fixed_maxsz + 1;
/* Skip Write list. */
while (*p++ != xdr_zero) {
nsegs = be32_to_cpup(p++);
p += nsegs * rpcrdma_segment_maxsz;
}
/* Skip Reply chunk. */
if (*p++ != xdr_zero) {
nsegs = be32_to_cpup(p++);
p += nsegs * rpcrdma_segment_maxsz;
}
return (unsigned long)p - (unsigned long)rdma_resp;
}
void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg *rmsgp, int chunks)
{
struct rpcrdma_write_array *ary;
/* no read-list */
rmsgp->rm_body.rm_chunks[0] = xdr_zero;
/* write-array discrim */
ary = (struct rpcrdma_write_array *)
&rmsgp->rm_body.rm_chunks[1];
ary->wc_discrim = xdr_one;
ary->wc_nchunks = cpu_to_be32(chunks);
/* write-list terminator */
ary->wc_array[chunks].wc_target.rs_handle = xdr_zero;
/* reply-array discriminator */
ary->wc_array[chunks].wc_target.rs_length = xdr_zero;
}
void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array *ary,
int chunks)
{
ary->wc_discrim = xdr_one;
ary->wc_nchunks = cpu_to_be32(chunks);
}
void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array *ary,
int chunk_no,
__be32 rs_handle,
__be64 rs_offset,
u32 write_len)
{
struct rpcrdma_segment *seg = &ary->wc_array[chunk_no].wc_target;
seg->rs_handle = rs_handle;
seg->rs_offset = rs_offset;
seg->rs_length = cpu_to_be32(write_len);
}
...@@ -558,33 +558,85 @@ static void rdma_read_complete(struct svc_rqst *rqstp, ...@@ -558,33 +558,85 @@ static void rdma_read_complete(struct svc_rqst *rqstp,
rqstp->rq_arg.buflen = head->arg.buflen; rqstp->rq_arg.buflen = head->arg.buflen;
} }
static void svc_rdma_send_error(struct svcxprt_rdma *xprt,
__be32 *rdma_argp, int status)
{
struct svc_rdma_op_ctxt *ctxt;
__be32 *p, *err_msgp;
unsigned int length;
struct page *page;
int ret;
ret = svc_rdma_repost_recv(xprt, GFP_KERNEL);
if (ret)
return;
page = alloc_page(GFP_KERNEL);
if (!page)
return;
err_msgp = page_address(page);
p = err_msgp;
*p++ = *rdma_argp;
*p++ = *(rdma_argp + 1);
*p++ = xprt->sc_fc_credits;
*p++ = rdma_error;
if (status == -EPROTONOSUPPORT) {
*p++ = err_vers;
*p++ = rpcrdma_version;
*p++ = rpcrdma_version;
} else {
*p++ = err_chunk;
}
length = (unsigned long)p - (unsigned long)err_msgp;
/* Map transport header; no RPC message payload */
ctxt = svc_rdma_get_context(xprt);
ret = svc_rdma_map_reply_hdr(xprt, ctxt, err_msgp, length);
if (ret) {
dprintk("svcrdma: Error %d mapping send for protocol error\n",
ret);
return;
}
ret = svc_rdma_post_send_wr(xprt, ctxt, 1, 0);
if (ret) {
dprintk("svcrdma: Error %d posting send for protocol error\n",
ret);
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 1);
}
}
/* By convention, backchannel calls arrive via rdma_msg type /* By convention, backchannel calls arrive via rdma_msg type
* messages, and never populate the chunk lists. This makes * messages, and never populate the chunk lists. This makes
* the RPC/RDMA header small and fixed in size, so it is * the RPC/RDMA header small and fixed in size, so it is
* straightforward to check the RPC header's direction field. * straightforward to check the RPC header's direction field.
*/ */
static bool static bool svc_rdma_is_backchannel_reply(struct svc_xprt *xprt,
svc_rdma_is_backchannel_reply(struct svc_xprt *xprt, struct rpcrdma_msg *rmsgp) __be32 *rdma_resp)
{ {
__be32 *p = (__be32 *)rmsgp; __be32 *p;
if (!xprt->xpt_bc_xprt) if (!xprt->xpt_bc_xprt)
return false; return false;
if (rmsgp->rm_type != rdma_msg) p = rdma_resp + 3;
if (*p++ != rdma_msg)
return false; return false;
if (rmsgp->rm_body.rm_chunks[0] != xdr_zero)
if (*p++ != xdr_zero)
return false; return false;
if (rmsgp->rm_body.rm_chunks[1] != xdr_zero) if (*p++ != xdr_zero)
return false; return false;
if (rmsgp->rm_body.rm_chunks[2] != xdr_zero) if (*p++ != xdr_zero)
return false; return false;
/* sanity */ /* XID sanity */
if (p[7] != rmsgp->rm_xid) if (*p++ != *rdma_resp)
return false; return false;
/* call direction */ /* call direction */
if (p[8] == cpu_to_be32(RPC_CALL)) if (*p == cpu_to_be32(RPC_CALL))
return false; return false;
return true; return true;
...@@ -650,8 +702,9 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) ...@@ -650,8 +702,9 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
goto out_drop; goto out_drop;
rqstp->rq_xprt_hlen = ret; rqstp->rq_xprt_hlen = ret;
if (svc_rdma_is_backchannel_reply(xprt, rmsgp)) { if (svc_rdma_is_backchannel_reply(xprt, &rmsgp->rm_xid)) {
ret = svc_rdma_handle_bc_reply(xprt->xpt_bc_xprt, rmsgp, ret = svc_rdma_handle_bc_reply(xprt->xpt_bc_xprt,
&rmsgp->rm_xid,
&rqstp->rq_arg); &rqstp->rq_arg);
svc_rdma_put_context(ctxt, 0); svc_rdma_put_context(ctxt, 0);
if (ret) if (ret)
...@@ -686,7 +739,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) ...@@ -686,7 +739,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
return ret; return ret;
out_err: out_err:
svc_rdma_send_error(rdma_xprt, rmsgp, ret); svc_rdma_send_error(rdma_xprt, &rmsgp->rm_xid, ret);
svc_rdma_put_context(ctxt, 0); svc_rdma_put_context(ctxt, 0);
return 0; return 0;
......
This diff is collapsed.
This diff is collapsed.
...@@ -272,85 +272,6 @@ static void svc_rdma_destroy_ctxts(struct svcxprt_rdma *xprt) ...@@ -272,85 +272,6 @@ static void svc_rdma_destroy_ctxts(struct svcxprt_rdma *xprt)
} }
} }
static struct svc_rdma_req_map *alloc_req_map(gfp_t flags)
{
struct svc_rdma_req_map *map;
map = kmalloc(sizeof(*map), flags);
if (map)
INIT_LIST_HEAD(&map->free);
return map;
}
static bool svc_rdma_prealloc_maps(struct svcxprt_rdma *xprt)
{
unsigned int i;
/* One for each receive buffer on this connection. */
i = xprt->sc_max_requests;
while (i--) {
struct svc_rdma_req_map *map;
map = alloc_req_map(GFP_KERNEL);
if (!map) {
dprintk("svcrdma: No memory for request map\n");
return false;
}
list_add(&map->free, &xprt->sc_maps);
}
return true;
}
struct svc_rdma_req_map *svc_rdma_get_req_map(struct svcxprt_rdma *xprt)
{
struct svc_rdma_req_map *map = NULL;
spin_lock(&xprt->sc_map_lock);
if (list_empty(&xprt->sc_maps))
goto out_empty;
map = list_first_entry(&xprt->sc_maps,
struct svc_rdma_req_map, free);
list_del_init(&map->free);
spin_unlock(&xprt->sc_map_lock);
out:
map->count = 0;
return map;
out_empty:
spin_unlock(&xprt->sc_map_lock);
/* Pre-allocation amount was incorrect */
map = alloc_req_map(GFP_NOIO);
if (map)
goto out;
WARN_ONCE(1, "svcrdma: empty request map list?\n");
return NULL;
}
void svc_rdma_put_req_map(struct svcxprt_rdma *xprt,
struct svc_rdma_req_map *map)
{
spin_lock(&xprt->sc_map_lock);
list_add(&map->free, &xprt->sc_maps);
spin_unlock(&xprt->sc_map_lock);
}
static void svc_rdma_destroy_maps(struct svcxprt_rdma *xprt)
{
while (!list_empty(&xprt->sc_maps)) {
struct svc_rdma_req_map *map;
map = list_first_entry(&xprt->sc_maps,
struct svc_rdma_req_map, free);
list_del(&map->free);
kfree(map);
}
}
/* QP event handler */ /* QP event handler */
static void qp_event_handler(struct ib_event *event, void *context) static void qp_event_handler(struct ib_event *event, void *context)
{ {
...@@ -473,24 +394,6 @@ void svc_rdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) ...@@ -473,24 +394,6 @@ void svc_rdma_wc_send(struct ib_cq *cq, struct ib_wc *wc)
svc_rdma_put_context(ctxt, 1); svc_rdma_put_context(ctxt, 1);
} }
/**
* svc_rdma_wc_write - Invoked by RDMA provider for each polled Write WC
* @cq: completion queue
* @wc: completed WR
*
*/
void svc_rdma_wc_write(struct ib_cq *cq, struct ib_wc *wc)
{
struct ib_cqe *cqe = wc->wr_cqe;
struct svc_rdma_op_ctxt *ctxt;
svc_rdma_send_wc_common_put(cq, wc, "write");
ctxt = container_of(cqe, struct svc_rdma_op_ctxt, cqe);
svc_rdma_unmap_dma(ctxt);
svc_rdma_put_context(ctxt, 0);
}
/** /**
* svc_rdma_wc_reg - Invoked by RDMA provider for each polled FASTREG WC * svc_rdma_wc_reg - Invoked by RDMA provider for each polled FASTREG WC
* @cq: completion queue * @cq: completion queue
...@@ -561,14 +464,14 @@ static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *serv, ...@@ -561,14 +464,14 @@ static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *serv,
INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q); INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
INIT_LIST_HEAD(&cma_xprt->sc_frmr_q); INIT_LIST_HEAD(&cma_xprt->sc_frmr_q);
INIT_LIST_HEAD(&cma_xprt->sc_ctxts); INIT_LIST_HEAD(&cma_xprt->sc_ctxts);
INIT_LIST_HEAD(&cma_xprt->sc_maps); INIT_LIST_HEAD(&cma_xprt->sc_rw_ctxts);
init_waitqueue_head(&cma_xprt->sc_send_wait); init_waitqueue_head(&cma_xprt->sc_send_wait);
spin_lock_init(&cma_xprt->sc_lock); spin_lock_init(&cma_xprt->sc_lock);
spin_lock_init(&cma_xprt->sc_rq_dto_lock); spin_lock_init(&cma_xprt->sc_rq_dto_lock);
spin_lock_init(&cma_xprt->sc_frmr_q_lock); spin_lock_init(&cma_xprt->sc_frmr_q_lock);
spin_lock_init(&cma_xprt->sc_ctxt_lock); spin_lock_init(&cma_xprt->sc_ctxt_lock);
spin_lock_init(&cma_xprt->sc_map_lock); spin_lock_init(&cma_xprt->sc_rw_ctxt_lock);
/* /*
* Note that this implies that the underlying transport support * Note that this implies that the underlying transport support
...@@ -999,6 +902,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) ...@@ -999,6 +902,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
newxprt, newxprt->sc_cm_id); newxprt, newxprt->sc_cm_id);
dev = newxprt->sc_cm_id->device; dev = newxprt->sc_cm_id->device;
newxprt->sc_port_num = newxprt->sc_cm_id->port_num;
/* Qualify the transport resource defaults with the /* Qualify the transport resource defaults with the
* capabilities of this particular device */ * capabilities of this particular device */
...@@ -1014,13 +918,11 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) ...@@ -1014,13 +918,11 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
svcrdma_max_bc_requests); svcrdma_max_bc_requests);
newxprt->sc_rq_depth = newxprt->sc_max_requests + newxprt->sc_rq_depth = newxprt->sc_max_requests +
newxprt->sc_max_bc_requests; newxprt->sc_max_bc_requests;
newxprt->sc_sq_depth = RPCRDMA_SQ_DEPTH_MULT * newxprt->sc_rq_depth; newxprt->sc_sq_depth = newxprt->sc_rq_depth;
atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth);
if (!svc_rdma_prealloc_ctxts(newxprt)) if (!svc_rdma_prealloc_ctxts(newxprt))
goto errout; goto errout;
if (!svc_rdma_prealloc_maps(newxprt))
goto errout;
/* /*
* Limit ORD based on client limit, local device limit, and * Limit ORD based on client limit, local device limit, and
...@@ -1050,6 +952,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) ...@@ -1050,6 +952,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
memset(&qp_attr, 0, sizeof qp_attr); memset(&qp_attr, 0, sizeof qp_attr);
qp_attr.event_handler = qp_event_handler; qp_attr.event_handler = qp_event_handler;
qp_attr.qp_context = &newxprt->sc_xprt; qp_attr.qp_context = &newxprt->sc_xprt;
qp_attr.port_num = newxprt->sc_cm_id->port_num;
qp_attr.cap.max_rdma_ctxs = newxprt->sc_max_requests;
qp_attr.cap.max_send_wr = newxprt->sc_sq_depth; qp_attr.cap.max_send_wr = newxprt->sc_sq_depth;
qp_attr.cap.max_recv_wr = newxprt->sc_rq_depth; qp_attr.cap.max_recv_wr = newxprt->sc_rq_depth;
qp_attr.cap.max_send_sge = newxprt->sc_max_sge; qp_attr.cap.max_send_sge = newxprt->sc_max_sge;
...@@ -1248,8 +1152,8 @@ static void __svc_rdma_free(struct work_struct *work) ...@@ -1248,8 +1152,8 @@ static void __svc_rdma_free(struct work_struct *work)
} }
rdma_dealloc_frmr_q(rdma); rdma_dealloc_frmr_q(rdma);
svc_rdma_destroy_rw_ctxts(rdma);
svc_rdma_destroy_ctxts(rdma); svc_rdma_destroy_ctxts(rdma);
svc_rdma_destroy_maps(rdma);
/* Destroy the QP if present (not a listener) */ /* Destroy the QP if present (not a listener) */
if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment