Commit c73be61c authored by David Howells's avatar David Howells

pipe: Add general notification queue support

Make it possible to have a general notification queue built on top of a
standard pipe.  Notifications are 'spliced' into the pipe and then read
out.  splice(), vmsplice() and sendfile() are forbidden on pipes used for
notifications as post_one_notification() cannot take pipe->mutex.  This
means that notifications could be posted in between individual pipe
buffers, making iov_iter_revert() difficult to effect.

The way the notification queue is used is:

 (1) An application opens a pipe with a special flag and indicates the
     number of messages it wishes to be able to queue at once (this can
     only be set once):

	pipe2(fds, O_NOTIFICATION_PIPE);
	ioctl(fds[0], IOC_WATCH_QUEUE_SET_SIZE, queue_depth);

 (2) The application then uses poll() and read() as normal to extract data
     from the pipe.  read() will return multiple notifications if the
     buffer is big enough, but it will not split a notification across
     buffers - rather it will return a short read or EMSGSIZE.

     Notification messages include a length in the header so that the
     caller can split them up.

Each message has a header that describes it:

	struct watch_notification {
		__u32	type:24;
		__u32	subtype:8;
		__u32	info;
	};

The type indicates the source (eg. mount tree changes, superblock events,
keyring changes, block layer events) and the subtype indicates the event
type (eg. mount, unmount; EIO, EDQUOT; link, unlink).  The info field
indicates a number of things, including the entry length, an ID assigned to
a watchpoint contributing to this buffer and type-specific flags.

Supplementary data, such as the key ID that generated an event, can be
attached in additional slots.  The maximum message size is 127 bytes.
Messages may not be padded or aligned, so there is no guarantee, for
example, that the notification type will be on a 4-byte bounary.
Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
parent b580b936
......@@ -201,6 +201,7 @@ Code Seq# Include File Comments
'W' 00-1F linux/wanrouter.h conflict! (pre 3.9)
'W' 00-3F sound/asound.h conflict!
'W' 40-5F drivers/pci/switch/switchtec.c
'W' 60-61 linux/watch_queue.h
'X' all fs/xfs/xfs_fs.h, conflict!
fs/xfs/linux-2.6/xfs_ioctl32.h,
include/linux/falloc.h,
......
This diff is collapsed.
......@@ -24,6 +24,7 @@
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/memcontrol.h>
#include <linux/watch_queue.h>
#include <linux/uaccess.h>
#include <asm/ioctls.h>
......@@ -459,6 +460,13 @@ pipe_write(struct kiocb *iocb, struct iov_iter *from)
goto out;
}
#ifdef CONFIG_WATCH_QUEUE
if (pipe->watch_queue) {
ret = -EXDEV;
goto out;
}
#endif
/*
* Only wake up if the pipe started out empty, since
* otherwise there should be no readers waiting.
......@@ -628,22 +636,37 @@ static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
int count, head, tail, mask;
switch (cmd) {
case FIONREAD:
__pipe_lock(pipe);
count = 0;
head = pipe->head;
tail = pipe->tail;
mask = pipe->ring_size - 1;
case FIONREAD:
__pipe_lock(pipe);
count = 0;
head = pipe->head;
tail = pipe->tail;
mask = pipe->ring_size - 1;
while (tail != head) {
count += pipe->bufs[tail & mask].len;
tail++;
}
__pipe_unlock(pipe);
while (tail != head) {
count += pipe->bufs[tail & mask].len;
tail++;
}
__pipe_unlock(pipe);
return put_user(count, (int __user *)arg);
default:
return -ENOIOCTLCMD;
return put_user(count, (int __user *)arg);
#ifdef CONFIG_WATCH_QUEUE
case IOC_WATCH_QUEUE_SET_SIZE: {
int ret;
__pipe_lock(pipe);
ret = watch_queue_set_size(pipe, arg);
__pipe_unlock(pipe);
return ret;
}
case IOC_WATCH_QUEUE_SET_FILTER:
return watch_queue_set_filter(
pipe, (struct watch_notification_filter __user *)arg);
#endif
default:
return -ENOIOCTLCMD;
}
}
......@@ -754,27 +777,27 @@ pipe_fasync(int fd, struct file *filp, int on)
return retval;
}
static unsigned long account_pipe_buffers(struct user_struct *user,
unsigned long old, unsigned long new)
unsigned long account_pipe_buffers(struct user_struct *user,
unsigned long old, unsigned long new)
{
return atomic_long_add_return(new - old, &user->pipe_bufs);
}
static bool too_many_pipe_buffers_soft(unsigned long user_bufs)
bool too_many_pipe_buffers_soft(unsigned long user_bufs)
{
unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
return soft_limit && user_bufs > soft_limit;
}
static bool too_many_pipe_buffers_hard(unsigned long user_bufs)
bool too_many_pipe_buffers_hard(unsigned long user_bufs)
{
unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
return hard_limit && user_bufs > hard_limit;
}
static bool is_unprivileged_user(void)
bool pipe_is_unprivileged_user(void)
{
return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
}
......@@ -796,12 +819,12 @@ struct pipe_inode_info *alloc_pipe_info(void)
user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
if (too_many_pipe_buffers_soft(user_bufs) && is_unprivileged_user()) {
if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
user_bufs = account_pipe_buffers(user, pipe_bufs, 1);
pipe_bufs = 1;
}
if (too_many_pipe_buffers_hard(user_bufs) && is_unprivileged_user())
if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
goto out_revert_acct;
pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
......@@ -813,6 +836,7 @@ struct pipe_inode_info *alloc_pipe_info(void)
pipe->r_counter = pipe->w_counter = 1;
pipe->max_usage = pipe_bufs;
pipe->ring_size = pipe_bufs;
pipe->nr_accounted = pipe_bufs;
pipe->user = user;
mutex_init(&pipe->mutex);
return pipe;
......@@ -830,7 +854,14 @@ void free_pipe_info(struct pipe_inode_info *pipe)
{
int i;
(void) account_pipe_buffers(pipe->user, pipe->ring_size, 0);
#ifdef CONFIG_WATCH_QUEUE
if (pipe->watch_queue) {
watch_queue_clear(pipe->watch_queue);
put_watch_queue(pipe->watch_queue);
}
#endif
(void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
free_uid(pipe->user);
for (i = 0; i < pipe->ring_size; i++) {
struct pipe_buffer *buf = pipe->bufs + i;
......@@ -906,6 +937,17 @@ int create_pipe_files(struct file **res, int flags)
if (!inode)
return -ENFILE;
if (flags & O_NOTIFICATION_PIPE) {
#ifdef CONFIG_WATCH_QUEUE
if (watch_queue_init(inode->i_pipe) < 0) {
iput(inode);
return -ENOMEM;
}
#else
return -ENOPKG;
#endif
}
f = alloc_file_pseudo(inode, pipe_mnt, "",
O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
&pipefifo_fops);
......@@ -936,7 +978,7 @@ static int __do_pipe_flags(int *fd, struct file **files, int flags)
int error;
int fdw, fdr;
if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT))
if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
return -EINVAL;
error = create_pipe_files(files, flags);
......@@ -1184,42 +1226,12 @@ unsigned int round_pipe_size(unsigned long size)
}
/*
* Allocate a new array of pipe buffers and copy the info over. Returns the
* pipe size if successful, or return -ERROR on error.
* Resize the pipe ring to a number of slots.
*/
static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
{
struct pipe_buffer *bufs;
unsigned int size, nr_slots, head, tail, mask, n;
unsigned long user_bufs;
long ret = 0;
size = round_pipe_size(arg);
nr_slots = size >> PAGE_SHIFT;
if (!nr_slots)
return -EINVAL;
/*
* If trying to increase the pipe capacity, check that an
* unprivileged user is not trying to exceed various limits
* (soft limit check here, hard limit check just below).
* Decreasing the pipe capacity is always permitted, even
* if the user is currently over a limit.
*/
if (nr_slots > pipe->ring_size &&
size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
return -EPERM;
user_bufs = account_pipe_buffers(pipe->user, pipe->ring_size, nr_slots);
if (nr_slots > pipe->ring_size &&
(too_many_pipe_buffers_hard(user_bufs) ||
too_many_pipe_buffers_soft(user_bufs)) &&
is_unprivileged_user()) {
ret = -EPERM;
goto out_revert_acct;
}
unsigned int head, tail, mask, n;
/*
* We can shrink the pipe, if arg is greater than the ring occupancy.
......@@ -1231,17 +1243,13 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
head = pipe->head;
tail = pipe->tail;
n = pipe_occupancy(pipe->head, pipe->tail);
if (nr_slots < n) {
ret = -EBUSY;
goto out_revert_acct;
}
if (nr_slots < n)
return -EBUSY;
bufs = kcalloc(nr_slots, sizeof(*bufs),
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (unlikely(!bufs)) {
ret = -ENOMEM;
goto out_revert_acct;
}
if (unlikely(!bufs))
return -ENOMEM;
/*
* The pipe array wraps around, so just start the new one at zero
......@@ -1269,16 +1277,68 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
kfree(pipe->bufs);
pipe->bufs = bufs;
pipe->ring_size = nr_slots;
pipe->max_usage = nr_slots;
if (pipe->max_usage > nr_slots)
pipe->max_usage = nr_slots;
pipe->tail = tail;
pipe->head = head;
/* This might have made more room for writers */
wake_up_interruptible(&pipe->wr_wait);
return 0;
}
/*
* Allocate a new array of pipe buffers and copy the info over. Returns the
* pipe size if successful, or return -ERROR on error.
*/
static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
{
unsigned long user_bufs;
unsigned int nr_slots, size;
long ret = 0;
#ifdef CONFIG_WATCH_QUEUE
if (pipe->watch_queue)
return -EBUSY;
#endif
size = round_pipe_size(arg);
nr_slots = size >> PAGE_SHIFT;
if (!nr_slots)
return -EINVAL;
/*
* If trying to increase the pipe capacity, check that an
* unprivileged user is not trying to exceed various limits
* (soft limit check here, hard limit check just below).
* Decreasing the pipe capacity is always permitted, even
* if the user is currently over a limit.
*/
if (nr_slots > pipe->max_usage &&
size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
return -EPERM;
user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
if (nr_slots > pipe->max_usage &&
(too_many_pipe_buffers_hard(user_bufs) ||
too_many_pipe_buffers_soft(user_bufs)) &&
pipe_is_unprivileged_user()) {
ret = -EPERM;
goto out_revert_acct;
}
ret = pipe_resize_ring(pipe, nr_slots);
if (ret < 0)
goto out_revert_acct;
pipe->max_usage = nr_slots;
pipe->nr_accounted = nr_slots;
return pipe->max_usage * PAGE_SIZE;
out_revert_acct:
(void) account_pipe_buffers(pipe->user, nr_slots, pipe->ring_size);
(void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
return ret;
}
......@@ -1287,9 +1347,17 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
* location, so checking ->i_pipe is not enough to verify that this is a
* pipe.
*/
struct pipe_inode_info *get_pipe_info(struct file *file)
struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
{
return file->f_op == &pipefifo_fops ? file->private_data : NULL;
struct pipe_inode_info *pipe = file->private_data;
if (file->f_op != &pipefifo_fops || !pipe)
return NULL;
#ifdef CONFIG_WATCH_QUEUE
if (for_splice && pipe->watch_queue)
return NULL;
#endif
return pipe;
}
long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
......@@ -1297,7 +1365,7 @@ long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
struct pipe_inode_info *pipe;
long ret;
pipe = get_pipe_info(file);
pipe = get_pipe_info(file, false);
if (!pipe)
return -EBADF;
......
......@@ -1122,8 +1122,8 @@ long do_splice(struct file *in, loff_t __user *off_in,
!(out->f_mode & FMODE_WRITE)))
return -EBADF;
ipipe = get_pipe_info(in);
opipe = get_pipe_info(out);
ipipe = get_pipe_info(in, true);
opipe = get_pipe_info(out, true);
if (ipipe && opipe) {
if (off_in || off_out)
......@@ -1273,7 +1273,7 @@ static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
unsigned int flags)
{
struct pipe_inode_info *pipe = get_pipe_info(file);
struct pipe_inode_info *pipe = get_pipe_info(file, true);
struct splice_desc sd = {
.total_len = iov_iter_count(iter),
.flags = flags,
......@@ -1308,7 +1308,7 @@ static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
if (flags & SPLICE_F_GIFT)
buf_flag = PIPE_BUF_FLAG_GIFT;
pipe = get_pipe_info(file);
pipe = get_pipe_info(file, true);
if (!pipe)
return -EBADF;
......@@ -1757,8 +1757,8 @@ static int link_pipe(struct pipe_inode_info *ipipe,
static long do_tee(struct file *in, struct file *out, size_t len,
unsigned int flags)
{
struct pipe_inode_info *ipipe = get_pipe_info(in);
struct pipe_inode_info *opipe = get_pipe_info(out);
struct pipe_inode_info *ipipe = get_pipe_info(in, true);
struct pipe_inode_info *opipe = get_pipe_info(out, true);
int ret = -EINVAL;
if (unlikely(!(in->f_mode & FMODE_READ) ||
......
......@@ -35,6 +35,7 @@ struct pipe_buffer {
* @tail: The point of buffer consumption
* @max_usage: The maximum number of slots that may be used in the ring
* @ring_size: total number of buffers (should be a power of 2)
* @nr_accounted: The amount this pipe accounts for in user->pipe_bufs
* @tmp_page: cached released page
* @readers: number of current readers of this pipe
* @writers: number of current writers of this pipe
......@@ -45,6 +46,7 @@ struct pipe_buffer {
* @fasync_writers: writer side fasync
* @bufs: the circular array of pipe buffers
* @user: the user who created this pipe
* @watch_queue: If this pipe is a watch_queue, this is the stuff for that
**/
struct pipe_inode_info {
struct mutex mutex;
......@@ -53,6 +55,7 @@ struct pipe_inode_info {
unsigned int tail;
unsigned int max_usage;
unsigned int ring_size;
unsigned int nr_accounted;
unsigned int readers;
unsigned int writers;
unsigned int files;
......@@ -63,6 +66,9 @@ struct pipe_inode_info {
struct fasync_struct *fasync_writers;
struct pipe_buffer *bufs;
struct user_struct *user;
#ifdef CONFIG_WATCH_QUEUE
struct watch_queue *watch_queue;
#endif
};
/*
......@@ -237,9 +243,20 @@ void pipe_buf_mark_unmergeable(struct pipe_buffer *buf);
extern const struct pipe_buf_operations nosteal_pipe_buf_ops;
#ifdef CONFIG_WATCH_QUEUE
unsigned long account_pipe_buffers(struct user_struct *user,
unsigned long old, unsigned long new);
bool too_many_pipe_buffers_soft(unsigned long user_bufs);
bool too_many_pipe_buffers_hard(unsigned long user_bufs);
bool pipe_is_unprivileged_user(void);
#endif
/* for F_SETPIPE_SZ and F_GETPIPE_SZ */
#ifdef CONFIG_WATCH_QUEUE
int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots);
#endif
long pipe_fcntl(struct file *, unsigned int, unsigned long arg);
struct pipe_inode_info *get_pipe_info(struct file *file);
struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice);
int create_pipe_files(struct file **, int);
unsigned int round_pipe_size(unsigned long size);
......
// SPDX-License-Identifier: GPL-2.0
/* User-mappable watch queue
*
* Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*
* See Documentation/watch_queue.rst
*/
#ifndef _LINUX_WATCH_QUEUE_H
#define _LINUX_WATCH_QUEUE_H
#include <uapi/linux/watch_queue.h>
#include <linux/kref.h>
#include <linux/rcupdate.h>
#ifdef CONFIG_WATCH_QUEUE
struct cred;
struct watch_type_filter {
enum watch_notification_type type;
__u32 subtype_filter[1]; /* Bitmask of subtypes to filter on */
__u32 info_filter; /* Filter on watch_notification::info */
__u32 info_mask; /* Mask of relevant bits in info_filter */
};
struct watch_filter {
union {
struct rcu_head rcu;
unsigned long type_filter[2]; /* Bitmask of accepted types */
};
u32 nr_filters; /* Number of filters */
struct watch_type_filter filters[];
};
struct watch_queue {
struct rcu_head rcu;
struct watch_filter __rcu *filter;
struct pipe_inode_info *pipe; /* The pipe we're using as a buffer */
struct hlist_head watches; /* Contributory watches */
struct page **notes; /* Preallocated notifications */
unsigned long *notes_bitmap; /* Allocation bitmap for notes */
struct kref usage; /* Object usage count */
spinlock_t lock;
unsigned int nr_notes; /* Number of notes */
unsigned int nr_pages; /* Number of pages in notes[] */
bool defunct; /* T when queues closed */
};
/*
* Representation of a watch on an object.
*/
struct watch {
union {
struct rcu_head rcu;
u32 info_id; /* ID to be OR'd in to info field */
};
struct watch_queue __rcu *queue; /* Queue to post events to */
struct hlist_node queue_node; /* Link in queue->watches */
struct watch_list __rcu *watch_list;
struct hlist_node list_node; /* Link in watch_list->watchers */
const struct cred *cred; /* Creds of the owner of the watch */
void *private; /* Private data for the watched object */
u64 id; /* Internal identifier */
struct kref usage; /* Object usage count */
};
/*
* List of watches on an object.
*/
struct watch_list {
struct rcu_head rcu;
struct hlist_head watchers;
void (*release_watch)(struct watch *);
spinlock_t lock;
};
extern void __post_watch_notification(struct watch_list *,
struct watch_notification *,
const struct cred *,
u64);
extern struct watch_queue *get_watch_queue(int);
extern void put_watch_queue(struct watch_queue *);
extern void init_watch(struct watch *, struct watch_queue *);
extern int add_watch_to_object(struct watch *, struct watch_list *);
extern int remove_watch_from_object(struct watch_list *, struct watch_queue *, u64, bool);
extern long watch_queue_set_size(struct pipe_inode_info *, unsigned int);
extern long watch_queue_set_filter(struct pipe_inode_info *,
struct watch_notification_filter __user *);
extern int watch_queue_init(struct pipe_inode_info *);
extern void watch_queue_clear(struct watch_queue *);
static inline void init_watch_list(struct watch_list *wlist,
void (*release_watch)(struct watch *))
{
INIT_HLIST_HEAD(&wlist->watchers);
spin_lock_init(&wlist->lock);
wlist->release_watch = release_watch;
}
static inline void post_watch_notification(struct watch_list *wlist,
struct watch_notification *n,
const struct cred *cred,
u64 id)
{
if (unlikely(wlist))
__post_watch_notification(wlist, n, cred, id);
}
static inline void remove_watch_list(struct watch_list *wlist, u64 id)
{
if (wlist) {
remove_watch_from_object(wlist, NULL, id, true);
kfree_rcu(wlist, rcu);
}
}
/**
* watch_sizeof - Calculate the information part of the size of a watch record,
* given the structure size.
*/
#define watch_sizeof(STRUCT) (sizeof(STRUCT) << WATCH_INFO_LENGTH__SHIFT)
#endif
#endif /* _LINUX_WATCH_QUEUE_H */
......@@ -4,9 +4,13 @@
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/ioctl.h>
#define O_NOTIFICATION_PIPE O_EXCL /* Parameter to pipe2() selecting notification pipe */
#define IOC_WATCH_QUEUE_SET_SIZE _IO('W', 0x60) /* Set the size in pages */
#define IOC_WATCH_QUEUE_SET_FILTER _IO('W', 0x61) /* Set the filter */
enum watch_notification_type {
WATCH_TYPE_META = 0, /* Special record */
WATCH_TYPE__NR = 1
......@@ -41,6 +45,22 @@ struct watch_notification {
#define WATCH_INFO_FLAG_7 0x00800000
};
/*
* Notification filtering rules (IOC_WATCH_QUEUE_SET_FILTER).
*/
struct watch_notification_type_filter {
__u32 type; /* Type to apply filter to */
__u32 info_filter; /* Filter on watch_notification::info */
__u32 info_mask; /* Mask of relevant bits in info_filter */
__u32 subtype_filter[8]; /* Bitmask of subtypes to filter on */
};
struct watch_notification_filter {
__u32 nr_filters; /* Number of filters */
__u32 __reserved; /* Must be 0 */
struct watch_notification_type_filter filters[];
};
/*
* Extended watch removal notification. This is used optionally if the type
......
......@@ -326,6 +326,18 @@ config POSIX_MQUEUE_SYSCTL
depends on SYSCTL
default y
config WATCH_QUEUE
bool "General notification queue"
default n
help
This is a general notification queue for the kernel to pass events to
userspace by splicing them into pipes. It can be used in conjunction
with watches for key/keyring change notifications and device
notifications.
See Documentation/watch_queue.rst
config CROSS_MEMORY_ATTACH
bool "Enable process_vm_readv/writev syscalls"
depends on MMU
......
......@@ -115,6 +115,7 @@ obj-$(CONFIG_TORTURE_TEST) += torture.o
obj-$(CONFIG_HAS_IOMEM) += iomem.o
obj-$(CONFIG_RSEQ) += rseq.o
obj-$(CONFIG_WATCH_QUEUE) += watch_queue.o
obj-$(CONFIG_SYSCTL_KUNIT_TEST) += sysctl-test.o
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment