Commit 8cefc107 authored by David Howells's avatar David Howells

pipe: Use head and tail pointers for the ring, not cursor and length

Convert pipes to use head and tail pointers for the buffer ring rather than
pointer and length as the latter requires two atomic ops to update (or a
combined op) whereas the former only requires one.

 (1) The head pointer is the point at which production occurs and points to
     the slot in which the next buffer will be placed.  This is equivalent
     to pipe->curbuf + pipe->nrbufs.

     The head pointer belongs to the write-side.

 (2) The tail pointer is the point at which consumption occurs.  It points
     to the next slot to be consumed.  This is equivalent to pipe->curbuf.

     The tail pointer belongs to the read-side.

 (3) head and tail are allowed to run to UINT_MAX and wrap naturally.  They
     are only masked off when the array is being accessed, e.g.:

	pipe->bufs[head & mask]

     This means that it is not necessary to have a dead slot in the ring as
     head == tail isn't ambiguous.

 (4) The ring is empty if "head == tail".

     A helper, pipe_empty(), is provided for this.

 (5) The occupancy of the ring is "head - tail".

     A helper, pipe_occupancy(), is provided for this.

 (6) The number of free slots in the ring is "pipe->ring_size - occupancy".

     A helper, pipe_space_for_user() is provided to indicate how many slots
     userspace may use.

 (7) The ring is full if "head - tail >= pipe->ring_size".

     A helper, pipe_full(), is provided for this.
Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
parent f94df989
...@@ -919,6 +919,7 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe, ...@@ -919,6 +919,7 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
.pos = *ppos, .pos = *ppos,
.u.data = &sgl, .u.data = &sgl,
}; };
unsigned int occupancy;
/* /*
* Rproc_serial does not yet support splice. To support splice * Rproc_serial does not yet support splice. To support splice
...@@ -929,21 +930,18 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe, ...@@ -929,21 +930,18 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
if (is_rproc_serial(port->out_vq->vdev)) if (is_rproc_serial(port->out_vq->vdev))
return -EINVAL; return -EINVAL;
/*
* pipe->nrbufs == 0 means there are no data to transfer,
* so this returns just 0 for no data.
*/
pipe_lock(pipe); pipe_lock(pipe);
if (!pipe->nrbufs) { ret = 0;
ret = 0; if (pipe_empty(pipe->head, pipe->tail))
goto error_out; goto error_out;
}
ret = wait_port_writable(port, filp->f_flags & O_NONBLOCK); ret = wait_port_writable(port, filp->f_flags & O_NONBLOCK);
if (ret < 0) if (ret < 0)
goto error_out; goto error_out;
buf = alloc_buf(port->portdev->vdev, 0, pipe->nrbufs); occupancy = pipe_occupancy(pipe->head, pipe->tail);
buf = alloc_buf(port->portdev->vdev, 0, occupancy);
if (!buf) { if (!buf) {
ret = -ENOMEM; ret = -ENOMEM;
goto error_out; goto error_out;
...@@ -951,7 +949,7 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe, ...@@ -951,7 +949,7 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
sgl.n = 0; sgl.n = 0;
sgl.len = 0; sgl.len = 0;
sgl.size = pipe->nrbufs; sgl.size = occupancy;
sgl.sg = buf->sg; sgl.sg = buf->sg;
sg_init_table(sgl.sg, sgl.size); sg_init_table(sgl.sg, sgl.size);
ret = __splice_from_pipe(pipe, &sd, pipe_to_sg); ret = __splice_from_pipe(pipe, &sd, pipe_to_sg);
......
...@@ -703,7 +703,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs) ...@@ -703,7 +703,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
cs->pipebufs++; cs->pipebufs++;
cs->nr_segs--; cs->nr_segs--;
} else { } else {
if (cs->nr_segs == cs->pipe->buffers) if (cs->nr_segs >= cs->pipe->ring_size)
return -EIO; return -EIO;
page = alloc_page(GFP_HIGHUSER); page = alloc_page(GFP_HIGHUSER);
...@@ -879,7 +879,7 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, ...@@ -879,7 +879,7 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
struct pipe_buffer *buf; struct pipe_buffer *buf;
int err; int err;
if (cs->nr_segs == cs->pipe->buffers) if (cs->nr_segs >= cs->pipe->ring_size)
return -EIO; return -EIO;
err = unlock_request(cs->req); err = unlock_request(cs->req);
...@@ -1341,7 +1341,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, ...@@ -1341,7 +1341,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
if (!fud) if (!fud)
return -EPERM; return -EPERM;
bufs = kvmalloc_array(pipe->buffers, sizeof(struct pipe_buffer), bufs = kvmalloc_array(pipe->ring_size, sizeof(struct pipe_buffer),
GFP_KERNEL); GFP_KERNEL);
if (!bufs) if (!bufs)
return -ENOMEM; return -ENOMEM;
...@@ -1353,7 +1353,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, ...@@ -1353,7 +1353,7 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
if (ret < 0) if (ret < 0)
goto out; goto out;
if (pipe->nrbufs + cs.nr_segs > pipe->buffers) { if (pipe_occupancy(pipe->head, pipe->tail) + cs.nr_segs > pipe->ring_size) {
ret = -EIO; ret = -EIO;
goto out; goto out;
} }
...@@ -1935,6 +1935,7 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, ...@@ -1935,6 +1935,7 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
struct file *out, loff_t *ppos, struct file *out, loff_t *ppos,
size_t len, unsigned int flags) size_t len, unsigned int flags)
{ {
unsigned int head, tail, mask, count;
unsigned nbuf; unsigned nbuf;
unsigned idx; unsigned idx;
struct pipe_buffer *bufs; struct pipe_buffer *bufs;
...@@ -1949,8 +1950,12 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, ...@@ -1949,8 +1950,12 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
pipe_lock(pipe); pipe_lock(pipe);
bufs = kvmalloc_array(pipe->nrbufs, sizeof(struct pipe_buffer), head = pipe->head;
GFP_KERNEL); tail = pipe->tail;
mask = pipe->ring_size - 1;
count = head - tail;
bufs = kvmalloc_array(count, sizeof(struct pipe_buffer), GFP_KERNEL);
if (!bufs) { if (!bufs) {
pipe_unlock(pipe); pipe_unlock(pipe);
return -ENOMEM; return -ENOMEM;
...@@ -1958,8 +1963,8 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, ...@@ -1958,8 +1963,8 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
nbuf = 0; nbuf = 0;
rem = 0; rem = 0;
for (idx = 0; idx < pipe->nrbufs && rem < len; idx++) for (idx = tail; idx < head && rem < len; idx++)
rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len; rem += pipe->bufs[idx & mask].len;
ret = -EINVAL; ret = -EINVAL;
if (rem < len) if (rem < len)
...@@ -1970,16 +1975,16 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, ...@@ -1970,16 +1975,16 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
struct pipe_buffer *ibuf; struct pipe_buffer *ibuf;
struct pipe_buffer *obuf; struct pipe_buffer *obuf;
BUG_ON(nbuf >= pipe->buffers); BUG_ON(nbuf >= pipe->ring_size);
BUG_ON(!pipe->nrbufs); BUG_ON(tail == head);
ibuf = &pipe->bufs[pipe->curbuf]; ibuf = &pipe->bufs[tail & mask];
obuf = &bufs[nbuf]; obuf = &bufs[nbuf];
if (rem >= ibuf->len) { if (rem >= ibuf->len) {
*obuf = *ibuf; *obuf = *ibuf;
ibuf->ops = NULL; ibuf->ops = NULL;
pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); tail++;
pipe->nrbufs--; pipe->tail = tail;
} else { } else {
if (!pipe_buf_get(pipe, ibuf)) if (!pipe_buf_get(pipe, ibuf))
goto out_free; goto out_free;
......
This diff is collapsed.
This diff is collapsed.
...@@ -30,9 +30,9 @@ struct pipe_buffer { ...@@ -30,9 +30,9 @@ struct pipe_buffer {
* struct pipe_inode_info - a linux kernel pipe * struct pipe_inode_info - a linux kernel pipe
* @mutex: mutex protecting the whole thing * @mutex: mutex protecting the whole thing
* @wait: reader/writer wait point in case of empty/full pipe * @wait: reader/writer wait point in case of empty/full pipe
* @nrbufs: the number of non-empty pipe buffers in this pipe * @head: The point of buffer production
* @buffers: total number of buffers (should be a power of 2) * @tail: The point of buffer consumption
* @curbuf: the current pipe buffer entry * @ring_size: total number of buffers (should be a power of 2)
* @tmp_page: cached released page * @tmp_page: cached released page
* @readers: number of current readers of this pipe * @readers: number of current readers of this pipe
* @writers: number of current writers of this pipe * @writers: number of current writers of this pipe
...@@ -48,7 +48,9 @@ struct pipe_buffer { ...@@ -48,7 +48,9 @@ struct pipe_buffer {
struct pipe_inode_info { struct pipe_inode_info {
struct mutex mutex; struct mutex mutex;
wait_queue_head_t wait; wait_queue_head_t wait;
unsigned int nrbufs, curbuf, buffers; unsigned int head;
unsigned int tail;
unsigned int ring_size;
unsigned int readers; unsigned int readers;
unsigned int writers; unsigned int writers;
unsigned int files; unsigned int files;
...@@ -104,6 +106,56 @@ struct pipe_buf_operations { ...@@ -104,6 +106,56 @@ struct pipe_buf_operations {
bool (*get)(struct pipe_inode_info *, struct pipe_buffer *); bool (*get)(struct pipe_inode_info *, struct pipe_buffer *);
}; };
/**
* pipe_empty - Return true if the pipe is empty
* @head: The pipe ring head pointer
* @tail: The pipe ring tail pointer
*/
static inline bool pipe_empty(unsigned int head, unsigned int tail)
{
return head == tail;
}
/**
* pipe_occupancy - Return number of slots used in the pipe
* @head: The pipe ring head pointer
* @tail: The pipe ring tail pointer
*/
static inline unsigned int pipe_occupancy(unsigned int head, unsigned int tail)
{
return head - tail;
}
/**
* pipe_full - Return true if the pipe is full
* @head: The pipe ring head pointer
* @tail: The pipe ring tail pointer
* @limit: The maximum amount of slots available.
*/
static inline bool pipe_full(unsigned int head, unsigned int tail,
unsigned int limit)
{
return pipe_occupancy(head, tail) >= limit;
}
/**
* pipe_space_for_user - Return number of slots available to userspace
* @head: The pipe ring head pointer
* @tail: The pipe ring tail pointer
* @pipe: The pipe info structure
*/
static inline unsigned int pipe_space_for_user(unsigned int head, unsigned int tail,
struct pipe_inode_info *pipe)
{
unsigned int p_occupancy, p_space;
p_occupancy = pipe_occupancy(head, tail);
if (p_occupancy >= pipe->ring_size)
return 0;
p_space = pipe->ring_size - p_occupancy;
return p_space;
}
/** /**
* pipe_buf_get - get a reference to a pipe_buffer * pipe_buf_get - get a reference to a pipe_buffer
* @pipe: the pipe that the buffer belongs to * @pipe: the pipe that the buffer belongs to
......
...@@ -45,8 +45,8 @@ struct iov_iter { ...@@ -45,8 +45,8 @@ struct iov_iter {
union { union {
unsigned long nr_segs; unsigned long nr_segs;
struct { struct {
int idx; unsigned int head;
int start_idx; unsigned int start_head;
}; };
}; };
}; };
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment