Commit 888a6f77 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'core-rcu-for-linus' of...

Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (52 commits)
  sched: fix RCU lockdep splat from task_group()
  rcu: using ACCESS_ONCE() to observe the jiffies_stall/rnp->qsmask value
  sched: suppress RCU lockdep splat in task_fork_fair
  net: suppress RCU lockdep false positive in sock_update_classid
  rcu: move check from rcu_dereference_bh to rcu_read_lock_bh_held
  rcu: Add advice to PROVE_RCU_REPEATEDLY kernel config parameter
  rcu: Add tracing data to support queueing models
  rcu: fix sparse errors in rcutorture.c
  rcu: only one evaluation of arg in rcu_dereference_check() unless sparse
  kernel: Remove undead ifdef CONFIG_DEBUG_LOCK_ALLOC
  rcu: fix _oddness handling of verbose stall warnings
  rcu: performance fixes to TINY_PREEMPT_RCU callback checking
  rcu: upgrade stallwarn.txt documentation for CPU-bound RT processes
  vhost: add __rcu annotations
  rcu: add comment stating that list_empty() applies to RCU-protected lists
  rcu: apply TINY_PREEMPT_RCU read-side speedup to TREE_PREEMPT_RCU
  rcu: combine duplicate code, courtesy of CONFIG_PREEMPT_RCU
  rcu: Upgrade srcu_read_lock() docbook about SRCU grace periods
  rcu: document ways of stalling updates in low-memory situations
  rcu: repair code-duplication FIXMEs
  ...
parents 31b7eab2 6506cf6c
...@@ -1645,7 +1645,9 @@ the amount of locking which needs to be done. ...@@ -1645,7 +1645,9 @@ the amount of locking which needs to be done.
all the readers who were traversing the list when we deleted the all the readers who were traversing the list when we deleted the
element are finished. We use <function>call_rcu()</function> to element are finished. We use <function>call_rcu()</function> to
register a callback which will actually destroy the object once register a callback which will actually destroy the object once
the readers are finished. all pre-existing readers are finished. Alternatively,
<function>synchronize_rcu()</function> may be used to block until
all pre-existing are finished.
</para> </para>
<para> <para>
But how does Read Copy Update know when the readers are But how does Read Copy Update know when the readers are
...@@ -1714,7 +1716,7 @@ the amount of locking which needs to be done. ...@@ -1714,7 +1716,7 @@ the amount of locking which needs to be done.
- object_put(obj); - object_put(obj);
+ list_del_rcu(&amp;obj-&gt;list); + list_del_rcu(&amp;obj-&gt;list);
cache_num--; cache_num--;
+ call_rcu(&amp;obj-&gt;rcu, cache_delete_rcu, obj); + call_rcu(&amp;obj-&gt;rcu, cache_delete_rcu);
} }
/* Must be holding cache_lock */ /* Must be holding cache_lock */
...@@ -1725,14 +1727,6 @@ the amount of locking which needs to be done. ...@@ -1725,14 +1727,6 @@ the amount of locking which needs to be done.
if (++cache_num > MAX_CACHE_SIZE) { if (++cache_num > MAX_CACHE_SIZE) {
struct object *i, *outcast = NULL; struct object *i, *outcast = NULL;
list_for_each_entry(i, &amp;cache, list) { list_for_each_entry(i, &amp;cache, list) {
@@ -85,6 +94,7 @@
obj-&gt;popularity = 0;
atomic_set(&amp;obj-&gt;refcnt, 1); /* The cache holds a reference */
spin_lock_init(&amp;obj-&gt;lock);
+ INIT_RCU_HEAD(&amp;obj-&gt;rcu);
spin_lock_irqsave(&amp;cache_lock, flags);
__cache_add(obj);
@@ -104,12 +114,11 @@ @@ -104,12 +114,11 @@
struct object *cache_find(int id) struct object *cache_find(int id)
{ {
......
...@@ -218,13 +218,22 @@ over a rather long period of time, but improvements are always welcome! ...@@ -218,13 +218,22 @@ over a rather long period of time, but improvements are always welcome!
include: include:
a. Keeping a count of the number of data-structure elements a. Keeping a count of the number of data-structure elements
used by the RCU-protected data structure, including those used by the RCU-protected data structure, including
waiting for a grace period to elapse. Enforce a limit those waiting for a grace period to elapse. Enforce a
on this number, stalling updates as needed to allow limit on this number, stalling updates as needed to allow
previously deferred frees to complete. previously deferred frees to complete. Alternatively,
limit only the number awaiting deferred free rather than
Alternatively, limit only the number awaiting deferred the total number of elements.
free rather than the total number of elements.
One way to stall the updates is to acquire the update-side
mutex. (Don't try this with a spinlock -- other CPUs
spinning on the lock could prevent the grace period
from ever ending.) Another way to stall the updates
is for the updates to use a wrapper function around
the memory allocator, so that this wrapper function
simulates OOM when there is too much memory awaiting an
RCU grace period. There are of course many other
variations on this theme.
b. Limiting update rate. For example, if updates occur only b. Limiting update rate. For example, if updates occur only
once per hour, then no explicit rate limiting is required, once per hour, then no explicit rate limiting is required,
...@@ -365,3 +374,26 @@ over a rather long period of time, but improvements are always welcome! ...@@ -365,3 +374,26 @@ over a rather long period of time, but improvements are always welcome!
and the compiler to freely reorder code into and out of RCU and the compiler to freely reorder code into and out of RCU
read-side critical sections. It is the responsibility of the read-side critical sections. It is the responsibility of the
RCU update-side primitives to deal with this. RCU update-side primitives to deal with this.
17. Use CONFIG_PROVE_RCU, CONFIG_DEBUG_OBJECTS_RCU_HEAD, and
the __rcu sparse checks to validate your RCU code. These
can help find problems as follows:
CONFIG_PROVE_RCU: check that accesses to RCU-protected data
structures are carried out under the proper RCU
read-side critical section, while holding the right
combination of locks, or whatever other conditions
are appropriate.
CONFIG_DEBUG_OBJECTS_RCU_HEAD: check that you don't pass the
same object to call_rcu() (or friends) before an RCU
grace period has elapsed since the last time that you
passed that same object to call_rcu() (or friends).
__rcu sparse checks: tag the pointer to the RCU-protected data
structure with __rcu, and sparse will warn you if you
access that pointer without the services of one of the
variants of rcu_dereference().
These debugging aids can help you find problems that are
otherwise extremely difficult to spot.
...@@ -80,6 +80,24 @@ o A CPU looping with bottom halves disabled. This condition can ...@@ -80,6 +80,24 @@ o A CPU looping with bottom halves disabled. This condition can
o For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the kernel o For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the kernel
without invoking schedule(). without invoking schedule().
o A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might
happen to preempt a low-priority task in the middle of an RCU
read-side critical section. This is especially damaging if
that low-priority task is not permitted to run on any other CPU,
in which case the next RCU grace period can never complete, which
will eventually cause the system to run out of memory and hang.
While the system is in the process of running itself out of
memory, you might see stall-warning messages.
o A CPU-bound real-time task in a CONFIG_PREEMPT_RT kernel that
is running at a higher priority than the RCU softirq threads.
This will prevent RCU callbacks from ever being invoked,
and in a CONFIG_TREE_PREEMPT_RCU kernel will further prevent
RCU grace periods from ever completing. Either way, the
system will eventually run out of memory and hang. In the
CONFIG_TREE_PREEMPT_RCU case, you might see stall-warning
messages.
o A bug in the RCU implementation. o A bug in the RCU implementation.
o A hardware failure. This is quite unlikely, but has occurred o A hardware failure. This is quite unlikely, but has occurred
......
...@@ -125,6 +125,17 @@ o "b" is the batch limit for this CPU. If more than this number ...@@ -125,6 +125,17 @@ o "b" is the batch limit for this CPU. If more than this number
of RCU callbacks is ready to invoke, then the remainder will of RCU callbacks is ready to invoke, then the remainder will
be deferred. be deferred.
o "ci" is the number of RCU callbacks that have been invoked for
this CPU. Note that ci+ql is the number of callbacks that have
been registered in absence of CPU-hotplug activity.
o "co" is the number of RCU callbacks that have been orphaned due to
this CPU going offline.
o "ca" is the number of RCU callbacks that have been adopted due to
other CPUs going offline. Note that ci+co-ca+ql is the number of
RCU callbacks registered on this CPU.
There is also an rcu/rcudata.csv file with the same information in There is also an rcu/rcudata.csv file with the same information in
comma-separated-variable spreadsheet format. comma-separated-variable spreadsheet format.
...@@ -180,7 +191,7 @@ o "s" is the "signaled" state that drives force_quiescent_state()'s ...@@ -180,7 +191,7 @@ o "s" is the "signaled" state that drives force_quiescent_state()'s
o "jfq" is the number of jiffies remaining for this grace period o "jfq" is the number of jiffies remaining for this grace period
before force_quiescent_state() is invoked to help push things before force_quiescent_state() is invoked to help push things
along. Note that CPUs in dyntick-idle mode thoughout the grace along. Note that CPUs in dyntick-idle mode throughout the grace
period will not report on their own, but rather must be check by period will not report on their own, but rather must be check by
some other CPU via force_quiescent_state(). some other CPU via force_quiescent_state().
......
...@@ -28,7 +28,7 @@ struct evdev { ...@@ -28,7 +28,7 @@ struct evdev {
int minor; int minor;
struct input_handle handle; struct input_handle handle;
wait_queue_head_t wait; wait_queue_head_t wait;
struct evdev_client *grab; struct evdev_client __rcu *grab;
struct list_head client_list; struct list_head client_list;
spinlock_t client_lock; /* protects client_list */ spinlock_t client_lock; /* protects client_list */
struct mutex mutex; struct mutex mutex;
......
...@@ -127,7 +127,10 @@ static void handle_tx(struct vhost_net *net) ...@@ -127,7 +127,10 @@ static void handle_tx(struct vhost_net *net)
size_t len, total_len = 0; size_t len, total_len = 0;
int err, wmem; int err, wmem;
size_t hdr_size; size_t hdr_size;
struct socket *sock = rcu_dereference(vq->private_data); struct socket *sock;
sock = rcu_dereference_check(vq->private_data,
lockdep_is_held(&vq->mutex));
if (!sock) if (!sock)
return; return;
...@@ -582,7 +585,10 @@ static void vhost_net_disable_vq(struct vhost_net *n, ...@@ -582,7 +585,10 @@ static void vhost_net_disable_vq(struct vhost_net *n,
static void vhost_net_enable_vq(struct vhost_net *n, static void vhost_net_enable_vq(struct vhost_net *n,
struct vhost_virtqueue *vq) struct vhost_virtqueue *vq)
{ {
struct socket *sock = vq->private_data; struct socket *sock;
sock = rcu_dereference_protected(vq->private_data,
lockdep_is_held(&vq->mutex));
if (!sock) if (!sock)
return; return;
if (vq == n->vqs + VHOST_NET_VQ_TX) { if (vq == n->vqs + VHOST_NET_VQ_TX) {
...@@ -598,7 +604,8 @@ static struct socket *vhost_net_stop_vq(struct vhost_net *n, ...@@ -598,7 +604,8 @@ static struct socket *vhost_net_stop_vq(struct vhost_net *n,
struct socket *sock; struct socket *sock;
mutex_lock(&vq->mutex); mutex_lock(&vq->mutex);
sock = vq->private_data; sock = rcu_dereference_protected(vq->private_data,
lockdep_is_held(&vq->mutex));
vhost_net_disable_vq(n, vq); vhost_net_disable_vq(n, vq);
rcu_assign_pointer(vq->private_data, NULL); rcu_assign_pointer(vq->private_data, NULL);
mutex_unlock(&vq->mutex); mutex_unlock(&vq->mutex);
...@@ -736,7 +743,8 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) ...@@ -736,7 +743,8 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
} }
/* start polling new socket */ /* start polling new socket */
oldsock = vq->private_data; oldsock = rcu_dereference_protected(vq->private_data,
lockdep_is_held(&vq->mutex));
if (sock != oldsock) { if (sock != oldsock) {
vhost_net_disable_vq(n, vq); vhost_net_disable_vq(n, vq);
rcu_assign_pointer(vq->private_data, sock); rcu_assign_pointer(vq->private_data, sock);
......
...@@ -320,7 +320,7 @@ long vhost_dev_reset_owner(struct vhost_dev *dev) ...@@ -320,7 +320,7 @@ long vhost_dev_reset_owner(struct vhost_dev *dev)
vhost_dev_cleanup(dev); vhost_dev_cleanup(dev);
memory->nregions = 0; memory->nregions = 0;
dev->memory = memory; RCU_INIT_POINTER(dev->memory, memory);
return 0; return 0;
} }
...@@ -352,8 +352,9 @@ void vhost_dev_cleanup(struct vhost_dev *dev) ...@@ -352,8 +352,9 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
fput(dev->log_file); fput(dev->log_file);
dev->log_file = NULL; dev->log_file = NULL;
/* No one will access memory at this point */ /* No one will access memory at this point */
kfree(dev->memory); kfree(rcu_dereference_protected(dev->memory,
dev->memory = NULL; lockdep_is_held(&dev->mutex)));
RCU_INIT_POINTER(dev->memory, NULL);
if (dev->mm) if (dev->mm)
mmput(dev->mm); mmput(dev->mm);
dev->mm = NULL; dev->mm = NULL;
...@@ -440,14 +441,22 @@ static int vq_access_ok(unsigned int num, ...@@ -440,14 +441,22 @@ static int vq_access_ok(unsigned int num,
/* Caller should have device mutex but not vq mutex */ /* Caller should have device mutex but not vq mutex */
int vhost_log_access_ok(struct vhost_dev *dev) int vhost_log_access_ok(struct vhost_dev *dev)
{ {
return memory_access_ok(dev, dev->memory, 1); struct vhost_memory *mp;
mp = rcu_dereference_protected(dev->memory,
lockdep_is_held(&dev->mutex));
return memory_access_ok(dev, mp, 1);
} }
/* Verify access for write logging. */ /* Verify access for write logging. */
/* Caller should have vq mutex and device mutex */ /* Caller should have vq mutex and device mutex */
static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base) static int vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base)
{ {
return vq_memory_access_ok(log_base, vq->dev->memory, struct vhost_memory *mp;
mp = rcu_dereference_protected(vq->dev->memory,
lockdep_is_held(&vq->mutex));
return vq_memory_access_ok(log_base, mp,
vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) && vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&
(!vq->log_used || log_access_ok(log_base, vq->log_addr, (!vq->log_used || log_access_ok(log_base, vq->log_addr,
sizeof *vq->used + sizeof *vq->used +
...@@ -487,7 +496,8 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) ...@@ -487,7 +496,8 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
kfree(newmem); kfree(newmem);
return -EFAULT; return -EFAULT;
} }
oldmem = d->memory; oldmem = rcu_dereference_protected(d->memory,
lockdep_is_held(&d->mutex));
rcu_assign_pointer(d->memory, newmem); rcu_assign_pointer(d->memory, newmem);
synchronize_rcu(); synchronize_rcu();
kfree(oldmem); kfree(oldmem);
......
...@@ -106,7 +106,7 @@ struct vhost_virtqueue { ...@@ -106,7 +106,7 @@ struct vhost_virtqueue {
* vhost_work execution acts instead of rcu_read_lock() and the end of * vhost_work execution acts instead of rcu_read_lock() and the end of
* vhost_work execution acts instead of rcu_read_lock(). * vhost_work execution acts instead of rcu_read_lock().
* Writers use virtqueue mutex. */ * Writers use virtqueue mutex. */
void *private_data; void __rcu *private_data;
/* Log write descriptors */ /* Log write descriptors */
void __user *log_base; void __user *log_base;
struct vhost_log log[VHOST_NET_MAX_SG]; struct vhost_log log[VHOST_NET_MAX_SG];
...@@ -116,7 +116,7 @@ struct vhost_dev { ...@@ -116,7 +116,7 @@ struct vhost_dev {
/* Readers use RCU to access memory table pointer /* Readers use RCU to access memory table pointer
* log base pointer and features. * log base pointer and features.
* Writers use mutex below.*/ * Writers use mutex below.*/
struct vhost_memory *memory; struct vhost_memory __rcu *memory;
struct mm_struct *mm; struct mm_struct *mm;
struct mutex mutex; struct mutex mutex;
unsigned acked_features; unsigned acked_features;
...@@ -173,7 +173,11 @@ enum { ...@@ -173,7 +173,11 @@ enum {
static inline int vhost_has_feature(struct vhost_dev *dev, int bit) static inline int vhost_has_feature(struct vhost_dev *dev, int bit)
{ {
unsigned acked_features = rcu_dereference(dev->acked_features); unsigned acked_features;
acked_features =
rcu_dereference_index_check(dev->acked_features,
lockdep_is_held(&dev->mutex));
return acked_features & (1 << bit); return acked_features & (1 << bit);
} }
......
...@@ -75,7 +75,7 @@ struct cgroup_subsys_state { ...@@ -75,7 +75,7 @@ struct cgroup_subsys_state {
unsigned long flags; unsigned long flags;
/* ID for this css, if possible */ /* ID for this css, if possible */
struct css_id *id; struct css_id __rcu *id;
}; };
/* bits in struct cgroup_subsys_state flags field */ /* bits in struct cgroup_subsys_state flags field */
...@@ -205,7 +205,7 @@ struct cgroup { ...@@ -205,7 +205,7 @@ struct cgroup {
struct list_head children; /* my children */ struct list_head children; /* my children */
struct cgroup *parent; /* my parent */ struct cgroup *parent; /* my parent */
struct dentry *dentry; /* cgroup fs entry, RCU protected */ struct dentry __rcu *dentry; /* cgroup fs entry, RCU protected */
/* Private pointers for each registered subsystem */ /* Private pointers for each registered subsystem */
struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT]; struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT];
......
...@@ -16,7 +16,11 @@ ...@@ -16,7 +16,11 @@
# define __release(x) __context__(x,-1) # define __release(x) __context__(x,-1)
# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) # define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
# define __percpu __attribute__((noderef, address_space(3))) # define __percpu __attribute__((noderef, address_space(3)))
#ifdef CONFIG_SPARSE_RCU_POINTER
# define __rcu __attribute__((noderef, address_space(4)))
#else
# define __rcu # define __rcu
#endif
extern void __chk_user_ptr(const volatile void __user *); extern void __chk_user_ptr(const volatile void __user *);
extern void __chk_io_ptr(const volatile void __iomem *); extern void __chk_io_ptr(const volatile void __iomem *);
#else #else
......
...@@ -84,7 +84,7 @@ struct thread_group_cred { ...@@ -84,7 +84,7 @@ struct thread_group_cred {
atomic_t usage; atomic_t usage;
pid_t tgid; /* thread group process ID */ pid_t tgid; /* thread group process ID */
spinlock_t lock; spinlock_t lock;
struct key *session_keyring; /* keyring inherited over fork */ struct key __rcu *session_keyring; /* keyring inherited over fork */
struct key *process_keyring; /* keyring private to this process */ struct key *process_keyring; /* keyring private to this process */
struct rcu_head rcu; /* RCU deletion hook */ struct rcu_head rcu; /* RCU deletion hook */
}; };
......
...@@ -31,7 +31,7 @@ struct embedded_fd_set { ...@@ -31,7 +31,7 @@ struct embedded_fd_set {
struct fdtable { struct fdtable {
unsigned int max_fds; unsigned int max_fds;
struct file ** fd; /* current fd array */ struct file __rcu **fd; /* current fd array */
fd_set *close_on_exec; fd_set *close_on_exec;
fd_set *open_fds; fd_set *open_fds;
struct rcu_head rcu; struct rcu_head rcu;
...@@ -46,7 +46,7 @@ struct files_struct { ...@@ -46,7 +46,7 @@ struct files_struct {
* read mostly part * read mostly part
*/ */
atomic_t count; atomic_t count;
struct fdtable *fdt; struct fdtable __rcu *fdt;
struct fdtable fdtab; struct fdtable fdtab;
/* /*
* written part on a separate cache line in SMP * written part on a separate cache line in SMP
...@@ -55,7 +55,7 @@ struct files_struct { ...@@ -55,7 +55,7 @@ struct files_struct {
int next_fd; int next_fd;
struct embedded_fd_set close_on_exec_init; struct embedded_fd_set close_on_exec_init;
struct embedded_fd_set open_fds_init; struct embedded_fd_set open_fds_init;
struct file * fd_array[NR_OPEN_DEFAULT]; struct file __rcu * fd_array[NR_OPEN_DEFAULT];
}; };
#define rcu_dereference_check_fdtable(files, fdtfd) \ #define rcu_dereference_check_fdtable(files, fdtfd) \
......
...@@ -1384,7 +1384,7 @@ struct super_block { ...@@ -1384,7 +1384,7 @@ struct super_block {
* Saved mount options for lazy filesystems using * Saved mount options for lazy filesystems using
* generic_show_options() * generic_show_options()
*/ */
char *s_options; char __rcu *s_options;
}; };
extern struct timespec current_fs_time(struct super_block *sb); extern struct timespec current_fs_time(struct super_block *sb);
......
...@@ -129,8 +129,8 @@ struct blk_scsi_cmd_filter { ...@@ -129,8 +129,8 @@ struct blk_scsi_cmd_filter {
struct disk_part_tbl { struct disk_part_tbl {
struct rcu_head rcu_head; struct rcu_head rcu_head;
int len; int len;
struct hd_struct *last_lookup; struct hd_struct __rcu *last_lookup;
struct hd_struct *part[]; struct hd_struct __rcu *part[];
}; };
struct gendisk { struct gendisk {
...@@ -149,7 +149,7 @@ struct gendisk { ...@@ -149,7 +149,7 @@ struct gendisk {
* non-critical accesses use RCU. Always access through * non-critical accesses use RCU. Always access through
* helpers. * helpers.
*/ */
struct disk_part_tbl *part_tbl; struct disk_part_tbl __rcu *part_tbl;
struct hd_struct part0; struct hd_struct part0;
const struct block_device_operations *fops; const struct block_device_operations *fops;
......
...@@ -139,7 +139,7 @@ static inline void account_system_vtime(struct task_struct *tsk) ...@@ -139,7 +139,7 @@ static inline void account_system_vtime(struct task_struct *tsk)
#endif #endif
#if defined(CONFIG_NO_HZ) #if defined(CONFIG_NO_HZ)
#if defined(CONFIG_TINY_RCU) #if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
extern void rcu_enter_nohz(void); extern void rcu_enter_nohz(void);
extern void rcu_exit_nohz(void); extern void rcu_exit_nohz(void);
......
...@@ -50,14 +50,14 @@ ...@@ -50,14 +50,14 @@
struct idr_layer { struct idr_layer {
unsigned long bitmap; /* A zero bit means "space here" */ unsigned long bitmap; /* A zero bit means "space here" */
struct idr_layer *ary[1<<IDR_BITS]; struct idr_layer __rcu *ary[1<<IDR_BITS];
int count; /* When zero, we can release it */ int count; /* When zero, we can release it */
int layer; /* distance from leaf */ int layer; /* distance from leaf */
struct rcu_head rcu_head; struct rcu_head rcu_head;
}; };
struct idr { struct idr {
struct idr_layer *top; struct idr_layer __rcu *top;
struct idr_layer *id_free; struct idr_layer *id_free;
int layers; /* only valid without concurrent changes */ int layers; /* only valid without concurrent changes */
int id_free_cnt; int id_free_cnt;
......
...@@ -82,11 +82,17 @@ extern struct group_info init_groups; ...@@ -82,11 +82,17 @@ extern struct group_info init_groups;
# define CAP_INIT_BSET CAP_FULL_SET # define CAP_INIT_BSET CAP_FULL_SET
#ifdef CONFIG_TREE_PREEMPT_RCU #ifdef CONFIG_TREE_PREEMPT_RCU
#define INIT_TASK_RCU_TREE_PREEMPT() \
.rcu_blocked_node = NULL,
#else
#define INIT_TASK_RCU_TREE_PREEMPT(tsk)
#endif
#ifdef CONFIG_PREEMPT_RCU
#define INIT_TASK_RCU_PREEMPT(tsk) \ #define INIT_TASK_RCU_PREEMPT(tsk) \
.rcu_read_lock_nesting = 0, \ .rcu_read_lock_nesting = 0, \
.rcu_read_unlock_special = 0, \ .rcu_read_unlock_special = 0, \
.rcu_blocked_node = NULL, \ .rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry), \
.rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry), INIT_TASK_RCU_TREE_PREEMPT()
#else #else
#define INIT_TASK_RCU_PREEMPT(tsk) #define INIT_TASK_RCU_PREEMPT(tsk)
#endif #endif
...@@ -137,8 +143,8 @@ extern struct cred init_cred; ...@@ -137,8 +143,8 @@ extern struct cred init_cred;
.children = LIST_HEAD_INIT(tsk.children), \ .children = LIST_HEAD_INIT(tsk.children), \
.sibling = LIST_HEAD_INIT(tsk.sibling), \ .sibling = LIST_HEAD_INIT(tsk.sibling), \
.group_leader = &tsk, \ .group_leader = &tsk, \
.real_cred = &init_cred, \ RCU_INIT_POINTER(.real_cred, &init_cred), \
.cred = &init_cred, \ RCU_INIT_POINTER(.cred, &init_cred), \
.cred_guard_mutex = \ .cred_guard_mutex = \
__MUTEX_INITIALIZER(tsk.cred_guard_mutex), \ __MUTEX_INITIALIZER(tsk.cred_guard_mutex), \
.comm = "swapper", \ .comm = "swapper", \
......
...@@ -1196,7 +1196,7 @@ struct input_dev { ...@@ -1196,7 +1196,7 @@ struct input_dev {
int (*flush)(struct input_dev *dev, struct file *file); int (*flush)(struct input_dev *dev, struct file *file);
int (*event)(struct input_dev *dev, unsigned int type, unsigned int code, int value); int (*event)(struct input_dev *dev, unsigned int type, unsigned int code, int value);
struct input_handle *grab; struct input_handle __rcu *grab;
spinlock_t event_lock; spinlock_t event_lock;
struct mutex mutex; struct mutex mutex;
......
...@@ -53,7 +53,7 @@ struct io_context { ...@@ -53,7 +53,7 @@ struct io_context {
struct radix_tree_root radix_root; struct radix_tree_root radix_root;
struct hlist_head cic_list; struct hlist_head cic_list;
void *ioc_data; void __rcu *ioc_data;
}; };
static inline struct io_context *ioc_task_link(struct io_context *ioc) static inline struct io_context *ioc_task_link(struct io_context *ioc)
......
...@@ -178,8 +178,9 @@ struct key { ...@@ -178,8 +178,9 @@ struct key {
*/ */
union { union {
unsigned long value; unsigned long value;
void __rcu *rcudata;
void *data; void *data;
struct keyring_list *subscriptions; struct keyring_list __rcu *subscriptions;
} payload; } payload;
}; };
......
...@@ -205,7 +205,7 @@ struct kvm { ...@@ -205,7 +205,7 @@ struct kvm {
struct mutex irq_lock; struct mutex irq_lock;
#ifdef CONFIG_HAVE_KVM_IRQCHIP #ifdef CONFIG_HAVE_KVM_IRQCHIP
struct kvm_irq_routing_table *irq_routing; struct kvm_irq_routing_table __rcu *irq_routing;
struct hlist_head mask_notifier_list; struct hlist_head mask_notifier_list;
struct hlist_head irq_ack_notifier_list; struct hlist_head irq_ack_notifier_list;
#endif #endif
......
...@@ -299,7 +299,7 @@ struct mm_struct { ...@@ -299,7 +299,7 @@ struct mm_struct {
* new_owner->mm == mm * new_owner->mm == mm
* new_owner->alloc_lock is held * new_owner->alloc_lock is held
*/ */
struct task_struct *owner; struct task_struct __rcu *owner;
#endif #endif
#ifdef CONFIG_PROC_FS #ifdef CONFIG_PROC_FS
......
...@@ -185,7 +185,7 @@ struct nfs_inode { ...@@ -185,7 +185,7 @@ struct nfs_inode {
struct nfs4_cached_acl *nfs4_acl; struct nfs4_cached_acl *nfs4_acl;
/* NFSv4 state */ /* NFSv4 state */
struct list_head open_states; struct list_head open_states;
struct nfs_delegation *delegation; struct nfs_delegation __rcu *delegation;
fmode_t delegation_state; fmode_t delegation_state;
struct rw_semaphore rwsem; struct rw_semaphore rwsem;
#endif /* CONFIG_NFS_V4*/ #endif /* CONFIG_NFS_V4*/
......
...@@ -49,28 +49,28 @@ ...@@ -49,28 +49,28 @@
struct notifier_block { struct notifier_block {
int (*notifier_call)(struct notifier_block *, unsigned long, void *); int (*notifier_call)(struct notifier_block *, unsigned long, void *);
struct notifier_block *next; struct notifier_block __rcu *next;
int priority; int priority;
}; };
struct atomic_notifier_head { struct atomic_notifier_head {
spinlock_t lock; spinlock_t lock;
struct notifier_block *head; struct notifier_block __rcu *head;
}; };
struct blocking_notifier_head { struct blocking_notifier_head {
struct rw_semaphore rwsem; struct rw_semaphore rwsem;
struct notifier_block *head; struct notifier_block __rcu *head;
}; };
struct raw_notifier_head { struct raw_notifier_head {
struct notifier_block *head; struct notifier_block __rcu *head;
}; };
struct srcu_notifier_head { struct srcu_notifier_head {
struct mutex mutex; struct mutex mutex;
struct srcu_struct srcu; struct srcu_struct srcu;
struct notifier_block *head; struct notifier_block __rcu *head;
}; };
#define ATOMIC_INIT_NOTIFIER_HEAD(name) do { \ #define ATOMIC_INIT_NOTIFIER_HEAD(name) do { \
......
...@@ -47,6 +47,8 @@ static inline void *radix_tree_indirect_to_ptr(void *ptr) ...@@ -47,6 +47,8 @@ static inline void *radix_tree_indirect_to_ptr(void *ptr)
{ {
return (void *)((unsigned long)ptr & ~RADIX_TREE_INDIRECT_PTR); return (void *)((unsigned long)ptr & ~RADIX_TREE_INDIRECT_PTR);
} }
#define radix_tree_indirect_to_ptr(ptr) \
radix_tree_indirect_to_ptr((void __force *)(ptr))
static inline int radix_tree_is_indirect_ptr(void *ptr) static inline int radix_tree_is_indirect_ptr(void *ptr)
{ {
...@@ -61,7 +63,7 @@ static inline int radix_tree_is_indirect_ptr(void *ptr) ...@@ -61,7 +63,7 @@ static inline int radix_tree_is_indirect_ptr(void *ptr)
struct radix_tree_root { struct radix_tree_root {
unsigned int height; unsigned int height;
gfp_t gfp_mask; gfp_t gfp_mask;
struct radix_tree_node *rnode; struct radix_tree_node __rcu *rnode;
}; };
#define RADIX_TREE_INIT(mask) { \ #define RADIX_TREE_INIT(mask) { \
......
...@@ -9,6 +9,21 @@ ...@@ -9,6 +9,21 @@
#include <linux/list.h> #include <linux/list.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
/*
* Why is there no list_empty_rcu()? Because list_empty() serves this
* purpose. The list_empty() function fetches the RCU-protected pointer
* and compares it to the address of the list head, but neither dereferences
* this pointer itself nor provides this pointer to the caller. Therefore,
* it is not necessary to use rcu_dereference(), so that list_empty() can
* be used anywhere you would want to use a list_empty_rcu().
*/
/*
* return the ->next pointer of a list_head in an rcu safe
* way, we must not access it directly
*/
#define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
/* /*
* Insert a new entry between two known consecutive entries. * Insert a new entry between two known consecutive entries.
* *
...@@ -20,7 +35,7 @@ static inline void __list_add_rcu(struct list_head *new, ...@@ -20,7 +35,7 @@ static inline void __list_add_rcu(struct list_head *new,
{ {
new->next = next; new->next = next;
new->prev = prev; new->prev = prev;
rcu_assign_pointer(prev->next, new); rcu_assign_pointer(list_next_rcu(prev), new);
next->prev = new; next->prev = new;
} }
...@@ -138,7 +153,7 @@ static inline void list_replace_rcu(struct list_head *old, ...@@ -138,7 +153,7 @@ static inline void list_replace_rcu(struct list_head *old,
{ {
new->next = old->next; new->next = old->next;
new->prev = old->prev; new->prev = old->prev;
rcu_assign_pointer(new->prev->next, new); rcu_assign_pointer(list_next_rcu(new->prev), new);
new->next->prev = new; new->next->prev = new;
old->prev = LIST_POISON2; old->prev = LIST_POISON2;
} }
...@@ -193,7 +208,7 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -193,7 +208,7 @@ static inline void list_splice_init_rcu(struct list_head *list,
*/ */
last->next = at; last->next = at;
rcu_assign_pointer(head->next, first); rcu_assign_pointer(list_next_rcu(head), first);
first->prev = head; first->prev = head;
at->prev = last; at->prev = last;
} }
...@@ -208,7 +223,9 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -208,7 +223,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
* primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock(). * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
*/ */
#define list_entry_rcu(ptr, type, member) \ #define list_entry_rcu(ptr, type, member) \
container_of(rcu_dereference_raw(ptr), type, member) ({typeof (*ptr) __rcu *__ptr = (typeof (*ptr) __rcu __force *)ptr; \
container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \
})
/** /**
* list_first_entry_rcu - get the first element from a list * list_first_entry_rcu - get the first element from a list
...@@ -225,9 +242,9 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -225,9 +242,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
list_entry_rcu((ptr)->next, type, member) list_entry_rcu((ptr)->next, type, member)
#define __list_for_each_rcu(pos, head) \ #define __list_for_each_rcu(pos, head) \
for (pos = rcu_dereference_raw((head)->next); \ for (pos = rcu_dereference_raw(list_next_rcu(head)); \
pos != (head); \ pos != (head); \
pos = rcu_dereference_raw(pos->next)) pos = rcu_dereference_raw(list_next_rcu((pos)))
/** /**
* list_for_each_entry_rcu - iterate over rcu list of given type * list_for_each_entry_rcu - iterate over rcu list of given type
...@@ -257,9 +274,9 @@ static inline void list_splice_init_rcu(struct list_head *list, ...@@ -257,9 +274,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
* as long as the traversal is guarded by rcu_read_lock(). * as long as the traversal is guarded by rcu_read_lock().
*/ */
#define list_for_each_continue_rcu(pos, head) \ #define list_for_each_continue_rcu(pos, head) \
for ((pos) = rcu_dereference_raw((pos)->next); \ for ((pos) = rcu_dereference_raw(list_next_rcu(pos)); \
prefetch((pos)->next), (pos) != (head); \ prefetch((pos)->next), (pos) != (head); \
(pos) = rcu_dereference_raw((pos)->next)) (pos) = rcu_dereference_raw(list_next_rcu(pos)))
/** /**
* list_for_each_entry_continue_rcu - continue iteration over list of given type * list_for_each_entry_continue_rcu - continue iteration over list of given type
...@@ -314,12 +331,19 @@ static inline void hlist_replace_rcu(struct hlist_node *old, ...@@ -314,12 +331,19 @@ static inline void hlist_replace_rcu(struct hlist_node *old,
new->next = next; new->next = next;
new->pprev = old->pprev; new->pprev = old->pprev;
rcu_assign_pointer(*new->pprev, new); rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
if (next) if (next)
new->next->pprev = &new->next; new->next->pprev = &new->next;
old->pprev = LIST_POISON2; old->pprev = LIST_POISON2;
} }
/*
* return the first or the next element in an RCU protected hlist
*/
#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
#define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
#define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
/** /**
* hlist_add_head_rcu * hlist_add_head_rcu
* @n: the element to add to the hash list. * @n: the element to add to the hash list.
...@@ -346,7 +370,7 @@ static inline void hlist_add_head_rcu(struct hlist_node *n, ...@@ -346,7 +370,7 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
n->next = first; n->next = first;
n->pprev = &h->first; n->pprev = &h->first;
rcu_assign_pointer(h->first, n); rcu_assign_pointer(hlist_first_rcu(h), n);
if (first) if (first)
first->pprev = &n->next; first->pprev = &n->next;
} }
...@@ -374,7 +398,7 @@ static inline void hlist_add_before_rcu(struct hlist_node *n, ...@@ -374,7 +398,7 @@ static inline void hlist_add_before_rcu(struct hlist_node *n,
{ {
n->pprev = next->pprev; n->pprev = next->pprev;
n->next = next; n->next = next;
rcu_assign_pointer(*(n->pprev), n); rcu_assign_pointer(hlist_pprev_rcu(n), n);
next->pprev = &n->next; next->pprev = &n->next;
} }
...@@ -401,15 +425,15 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, ...@@ -401,15 +425,15 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
{ {
n->next = prev->next; n->next = prev->next;
n->pprev = &prev->next; n->pprev = &prev->next;
rcu_assign_pointer(prev->next, n); rcu_assign_pointer(hlist_next_rcu(prev), n);
if (n->next) if (n->next)
n->next->pprev = &n->next; n->next->pprev = &n->next;
} }
#define __hlist_for_each_rcu(pos, head) \ #define __hlist_for_each_rcu(pos, head) \
for (pos = rcu_dereference((head)->first); \ for (pos = rcu_dereference(hlist_first_rcu(head)); \
pos && ({ prefetch(pos->next); 1; }); \ pos && ({ prefetch(pos->next); 1; }); \
pos = rcu_dereference(pos->next)) pos = rcu_dereference(hlist_next_rcu(pos)))
/** /**
* hlist_for_each_entry_rcu - iterate over rcu list of given type * hlist_for_each_entry_rcu - iterate over rcu list of given type
...@@ -422,11 +446,11 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, ...@@ -422,11 +446,11 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
* the _rcu list-mutation primitives such as hlist_add_head_rcu() * the _rcu list-mutation primitives such as hlist_add_head_rcu()
* as long as the traversal is guarded by rcu_read_lock(). * as long as the traversal is guarded by rcu_read_lock().
*/ */
#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
for (pos = rcu_dereference_raw((head)->first); \ for (pos = rcu_dereference_raw(hlist_first_rcu(head)); \
pos && ({ prefetch(pos->next); 1; }) && \ pos && ({ prefetch(pos->next); 1; }) && \
({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
pos = rcu_dereference_raw(pos->next)) pos = rcu_dereference_raw(hlist_next_rcu(pos)))
/** /**
* hlist_for_each_entry_rcu_bh - iterate over rcu list of given type * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
......
...@@ -37,6 +37,12 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) ...@@ -37,6 +37,12 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n)
} }
} }
#define hlist_nulls_first_rcu(head) \
(*((struct hlist_nulls_node __rcu __force **)&(head)->first))
#define hlist_nulls_next_rcu(node) \
(*((struct hlist_nulls_node __rcu __force **)&(node)->next))
/** /**
* hlist_nulls_del_rcu - deletes entry from hash list without re-initialization * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization
* @n: the element to delete from the hash list. * @n: the element to delete from the hash list.
...@@ -88,7 +94,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, ...@@ -88,7 +94,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,
n->next = first; n->next = first;
n->pprev = &h->first; n->pprev = &h->first;
rcu_assign_pointer(h->first, n); rcu_assign_pointer(hlist_nulls_first_rcu(h), n);
if (!is_a_nulls(first)) if (!is_a_nulls(first))
first->pprev = &n->next; first->pprev = &n->next;
} }
...@@ -100,11 +106,11 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, ...@@ -100,11 +106,11 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,
* @member: the name of the hlist_nulls_node within the struct. * @member: the name of the hlist_nulls_node within the struct.
* *
*/ */
#define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \
for (pos = rcu_dereference_raw((head)->first); \ for (pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \
(!is_a_nulls(pos)) && \ (!is_a_nulls(pos)) && \
({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \
pos = rcu_dereference_raw(pos->next)) pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)))
#endif #endif
#endif #endif
This diff is collapsed.
...@@ -27,103 +27,101 @@ ...@@ -27,103 +27,101 @@
#include <linux/cache.h> #include <linux/cache.h>
void rcu_sched_qs(int cpu); #define rcu_init_sched() do { } while (0)
void rcu_bh_qs(int cpu);
static inline void rcu_note_context_switch(int cpu)
{
rcu_sched_qs(cpu);
}
#define __rcu_read_lock() preempt_disable() #ifdef CONFIG_TINY_RCU
#define __rcu_read_unlock() preempt_enable()
#define __rcu_read_lock_bh() local_bh_disable()
#define __rcu_read_unlock_bh() local_bh_enable()
#define call_rcu_sched call_rcu
#define rcu_init_sched() do { } while (0) static inline void synchronize_rcu_expedited(void)
extern void rcu_check_callbacks(int cpu, int user); {
synchronize_sched(); /* Only one CPU, so pretty fast anyway!!! */
}
static inline int rcu_needs_cpu(int cpu) static inline void rcu_barrier(void)
{ {
return 0; rcu_barrier_sched(); /* Only one CPU, so only one list of callbacks! */
} }
/* #else /* #ifdef CONFIG_TINY_RCU */
* Return the number of grace periods.
*/ void rcu_barrier(void);
static inline long rcu_batches_completed(void) void synchronize_rcu_expedited(void);
#endif /* #else #ifdef CONFIG_TINY_RCU */
static inline void synchronize_rcu_bh(void)
{ {
return 0; synchronize_sched();
} }
/* static inline void synchronize_rcu_bh_expedited(void)
* Return the number of bottom-half grace periods.
*/
static inline long rcu_batches_completed_bh(void)
{ {
return 0; synchronize_sched();
} }
static inline void rcu_force_quiescent_state(void) #ifdef CONFIG_TINY_RCU
static inline void rcu_preempt_note_context_switch(void)
{ {
} }
static inline void rcu_bh_force_quiescent_state(void) static inline void exit_rcu(void)
{ {
} }
static inline void rcu_sched_force_quiescent_state(void) static inline int rcu_needs_cpu(int cpu)
{ {
return 0;
} }
extern void synchronize_sched(void); #else /* #ifdef CONFIG_TINY_RCU */
void rcu_preempt_note_context_switch(void);
extern void exit_rcu(void);
int rcu_preempt_needs_cpu(void);
static inline void synchronize_rcu(void) static inline int rcu_needs_cpu(int cpu)
{ {
synchronize_sched(); return rcu_preempt_needs_cpu();
} }
static inline void synchronize_rcu_bh(void) #endif /* #else #ifdef CONFIG_TINY_RCU */
static inline void rcu_note_context_switch(int cpu)
{ {
synchronize_sched(); rcu_sched_qs(cpu);
rcu_preempt_note_context_switch();
} }
static inline void synchronize_rcu_expedited(void) /*
* Return the number of grace periods.
*/
static inline long rcu_batches_completed(void)
{ {
synchronize_sched(); return 0;
} }
static inline void synchronize_rcu_bh_expedited(void) /*
* Return the number of bottom-half grace periods.
*/
static inline long rcu_batches_completed_bh(void)
{ {
synchronize_sched(); return 0;
} }
struct notifier_block; static inline void rcu_force_quiescent_state(void)
#ifdef CONFIG_NO_HZ
extern void rcu_enter_nohz(void);
extern void rcu_exit_nohz(void);
#else /* #ifdef CONFIG_NO_HZ */
static inline void rcu_enter_nohz(void)
{ {
} }
static inline void rcu_exit_nohz(void) static inline void rcu_bh_force_quiescent_state(void)
{ {
} }
#endif /* #else #ifdef CONFIG_NO_HZ */ static inline void rcu_sched_force_quiescent_state(void)
static inline void exit_rcu(void)
{ {
} }
static inline int rcu_preempt_depth(void) static inline void rcu_cpu_stall_reset(void)
{ {
return 0;
} }
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
......
...@@ -30,64 +30,23 @@ ...@@ -30,64 +30,23 @@
#ifndef __LINUX_RCUTREE_H #ifndef __LINUX_RCUTREE_H
#define __LINUX_RCUTREE_H #define __LINUX_RCUTREE_H
struct notifier_block;
extern void rcu_sched_qs(int cpu);
extern void rcu_bh_qs(int cpu);
extern void rcu_note_context_switch(int cpu); extern void rcu_note_context_switch(int cpu);
extern int rcu_needs_cpu(int cpu); extern int rcu_needs_cpu(int cpu);
extern void rcu_cpu_stall_reset(void);
#ifdef CONFIG_TREE_PREEMPT_RCU #ifdef CONFIG_TREE_PREEMPT_RCU
extern void __rcu_read_lock(void);
extern void __rcu_read_unlock(void);
extern void synchronize_rcu(void);
extern void exit_rcu(void); extern void exit_rcu(void);
/*
* Defined as macro as it is a very low level header
* included from areas that don't even know about current
*/
#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */ #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
static inline void __rcu_read_lock(void)
{
preempt_disable();
}
static inline void __rcu_read_unlock(void)
{
preempt_enable();
}
#define synchronize_rcu synchronize_sched
static inline void exit_rcu(void) static inline void exit_rcu(void)
{ {
} }
static inline int rcu_preempt_depth(void)
{
return 0;
}
#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
static inline void __rcu_read_lock_bh(void)
{
local_bh_disable();
}
static inline void __rcu_read_unlock_bh(void)
{
local_bh_enable();
}
extern void call_rcu_sched(struct rcu_head *head,
void (*func)(struct rcu_head *rcu));
extern void synchronize_rcu_bh(void); extern void synchronize_rcu_bh(void);
extern void synchronize_sched(void);
extern void synchronize_rcu_expedited(void); extern void synchronize_rcu_expedited(void);
static inline void synchronize_rcu_bh_expedited(void) static inline void synchronize_rcu_bh_expedited(void)
...@@ -95,7 +54,7 @@ static inline void synchronize_rcu_bh_expedited(void) ...@@ -95,7 +54,7 @@ static inline void synchronize_rcu_bh_expedited(void)
synchronize_sched_expedited(); synchronize_sched_expedited();
} }
extern void rcu_check_callbacks(int cpu, int user); extern void rcu_barrier(void);
extern long rcu_batches_completed(void); extern long rcu_batches_completed(void);
extern long rcu_batches_completed_bh(void); extern long rcu_batches_completed_bh(void);
...@@ -104,18 +63,6 @@ extern void rcu_force_quiescent_state(void); ...@@ -104,18 +63,6 @@ extern void rcu_force_quiescent_state(void);
extern void rcu_bh_force_quiescent_state(void); extern void rcu_bh_force_quiescent_state(void);
extern void rcu_sched_force_quiescent_state(void); extern void rcu_sched_force_quiescent_state(void);
#ifdef CONFIG_NO_HZ
void rcu_enter_nohz(void);
void rcu_exit_nohz(void);
#else /* CONFIG_NO_HZ */
static inline void rcu_enter_nohz(void)
{
}
static inline void rcu_exit_nohz(void)
{
}
#endif /* CONFIG_NO_HZ */
/* A context switch is a grace period for RCU-sched and RCU-bh. */ /* A context switch is a grace period for RCU-sched and RCU-bh. */
static inline int rcu_blocking_is_gp(void) static inline int rcu_blocking_is_gp(void)
{ {
......
...@@ -1202,11 +1202,13 @@ struct task_struct { ...@@ -1202,11 +1202,13 @@ struct task_struct {
unsigned int policy; unsigned int policy;
cpumask_t cpus_allowed; cpumask_t cpus_allowed;
#ifdef CONFIG_TREE_PREEMPT_RCU #ifdef CONFIG_PREEMPT_RCU
int rcu_read_lock_nesting; int rcu_read_lock_nesting;
char rcu_read_unlock_special; char rcu_read_unlock_special;
struct rcu_node *rcu_blocked_node;
struct list_head rcu_node_entry; struct list_head rcu_node_entry;
#endif /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TREE_PREEMPT_RCU
struct rcu_node *rcu_blocked_node;
#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
...@@ -1288,9 +1290,9 @@ struct task_struct { ...@@ -1288,9 +1290,9 @@ struct task_struct {
struct list_head cpu_timers[3]; struct list_head cpu_timers[3];
/* process credentials */ /* process credentials */
const struct cred *real_cred; /* objective and real subjective task const struct cred __rcu *real_cred; /* objective and real subjective task
* credentials (COW) */ * credentials (COW) */
const struct cred *cred; /* effective (overridable) subjective task const struct cred __rcu *cred; /* effective (overridable) subjective task
* credentials (COW) */ * credentials (COW) */
struct mutex cred_guard_mutex; /* guard against foreign influences on struct mutex cred_guard_mutex; /* guard against foreign influences on
* credential calculations * credential calculations
...@@ -1418,7 +1420,7 @@ struct task_struct { ...@@ -1418,7 +1420,7 @@ struct task_struct {
#endif #endif
#ifdef CONFIG_CGROUPS #ifdef CONFIG_CGROUPS
/* Control Group info protected by css_set_lock */ /* Control Group info protected by css_set_lock */
struct css_set *cgroups; struct css_set __rcu *cgroups;
/* cg_list protected by css_set_lock and tsk->alloc_lock */ /* cg_list protected by css_set_lock and tsk->alloc_lock */
struct list_head cg_list; struct list_head cg_list;
#endif #endif
...@@ -1740,7 +1742,7 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t * ...@@ -1740,7 +1742,7 @@ extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *
#define tsk_used_math(p) ((p)->flags & PF_USED_MATH) #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
#define used_math() tsk_used_math(current) #define used_math() tsk_used_math(current)
#ifdef CONFIG_TREE_PREEMPT_RCU #ifdef CONFIG_PREEMPT_RCU
#define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */ #define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */
#define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */ #define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */
...@@ -1749,7 +1751,9 @@ static inline void rcu_copy_process(struct task_struct *p) ...@@ -1749,7 +1751,9 @@ static inline void rcu_copy_process(struct task_struct *p)
{ {
p->rcu_read_lock_nesting = 0; p->rcu_read_lock_nesting = 0;
p->rcu_read_unlock_special = 0; p->rcu_read_unlock_special = 0;
#ifdef CONFIG_TREE_PREEMPT_RCU
p->rcu_blocked_node = NULL; p->rcu_blocked_node = NULL;
#endif
INIT_LIST_HEAD(&p->rcu_node_entry); INIT_LIST_HEAD(&p->rcu_node_entry);
} }
......
...@@ -108,19 +108,43 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp) ...@@ -108,19 +108,43 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
/** /**
* srcu_dereference - fetch SRCU-protected pointer with checking * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
* @p: the pointer to fetch and protect for later dereferencing
* @sp: pointer to the srcu_struct, which is used to check that we
* really are in an SRCU read-side critical section.
* @c: condition to check for update-side use
* *
* Makes rcu_dereference_check() do the dirty work. * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
* critical section will result in an RCU-lockdep splat, unless @c evaluates
* to 1. The @c argument will normally be a logical expression containing
* lockdep_is_held() calls.
*/ */
#define srcu_dereference(p, sp) \ #define srcu_dereference_check(p, sp, c) \
rcu_dereference_check(p, srcu_read_lock_held(sp)) __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu)
/**
* srcu_dereference - fetch SRCU-protected pointer for later dereferencing
* @p: the pointer to fetch and protect for later dereferencing
* @sp: pointer to the srcu_struct, which is used to check that we
* really are in an SRCU read-side critical section.
*
* Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
* is enabled, invoking this outside of an RCU read-side critical
* section will result in an RCU-lockdep splat.
*/
#define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0)
/** /**
* srcu_read_lock - register a new reader for an SRCU-protected structure. * srcu_read_lock - register a new reader for an SRCU-protected structure.
* @sp: srcu_struct in which to register the new reader. * @sp: srcu_struct in which to register the new reader.
* *
* Enter an SRCU read-side critical section. Note that SRCU read-side * Enter an SRCU read-side critical section. Note that SRCU read-side
* critical sections may be nested. * critical sections may be nested. However, it is illegal to
* call anything that waits on an SRCU grace period for the same
* srcu_struct, whether directly or indirectly. Please note that
* one way to indirectly wait on an SRCU grace period is to acquire
* a mutex that is held elsewhere while calling synchronize_srcu() or
* synchronize_srcu_expedited().
*/ */
static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp)
{ {
......
...@@ -69,7 +69,7 @@ struct gss_cl_ctx { ...@@ -69,7 +69,7 @@ struct gss_cl_ctx {
enum rpc_gss_proc gc_proc; enum rpc_gss_proc gc_proc;
u32 gc_seq; u32 gc_seq;
spinlock_t gc_seq_lock; spinlock_t gc_seq_lock;
struct gss_ctx *gc_gss_ctx; struct gss_ctx __rcu *gc_gss_ctx;
struct xdr_netobj gc_wire_ctx; struct xdr_netobj gc_wire_ctx;
u32 gc_win; u32 gc_win;
unsigned long gc_expiry; unsigned long gc_expiry;
...@@ -80,7 +80,7 @@ struct gss_upcall_msg; ...@@ -80,7 +80,7 @@ struct gss_upcall_msg;
struct gss_cred { struct gss_cred {
struct rpc_cred gc_base; struct rpc_cred gc_base;
enum rpc_gss_svc gc_service; enum rpc_gss_svc gc_service;
struct gss_cl_ctx *gc_ctx; struct gss_cl_ctx __rcu *gc_ctx;
struct gss_upcall_msg *gc_upcall; struct gss_upcall_msg *gc_upcall;
unsigned long gc_upcall_timestamp; unsigned long gc_upcall_timestamp;
unsigned char gc_machine_cred : 1; unsigned char gc_machine_cred : 1;
......
...@@ -51,7 +51,8 @@ static inline u32 task_cls_classid(struct task_struct *p) ...@@ -51,7 +51,8 @@ static inline u32 task_cls_classid(struct task_struct *p)
return 0; return 0;
rcu_read_lock(); rcu_read_lock();
id = rcu_dereference(net_cls_subsys_id); id = rcu_dereference_index_check(net_cls_subsys_id,
rcu_read_lock_held());
if (id >= 0) if (id >= 0)
classid = container_of(task_subsys_state(p, id), classid = container_of(task_subsys_state(p, id),
struct cgroup_cls_state, css)->classid; struct cgroup_cls_state, css)->classid;
......
...@@ -75,7 +75,7 @@ struct nf_conntrack_helper; ...@@ -75,7 +75,7 @@ struct nf_conntrack_helper;
/* nf_conn feature for connections that have a helper */ /* nf_conn feature for connections that have a helper */
struct nf_conn_help { struct nf_conn_help {
/* Helper. if any */ /* Helper. if any */
struct nf_conntrack_helper *helper; struct nf_conntrack_helper __rcu *helper;
union nf_conntrack_help help; union nf_conntrack_help help;
......
...@@ -340,6 +340,7 @@ choice ...@@ -340,6 +340,7 @@ choice
config TREE_RCU config TREE_RCU
bool "Tree-based hierarchical RCU" bool "Tree-based hierarchical RCU"
depends on !PREEMPT && SMP
help help
This option selects the RCU implementation that is This option selects the RCU implementation that is
designed for very large SMP system with hundreds or designed for very large SMP system with hundreds or
...@@ -347,7 +348,7 @@ config TREE_RCU ...@@ -347,7 +348,7 @@ config TREE_RCU
smaller systems. smaller systems.
config TREE_PREEMPT_RCU config TREE_PREEMPT_RCU
bool "Preemptable tree-based hierarchical RCU" bool "Preemptible tree-based hierarchical RCU"
depends on PREEMPT depends on PREEMPT
help help
This option selects the RCU implementation that is This option selects the RCU implementation that is
...@@ -365,8 +366,22 @@ config TINY_RCU ...@@ -365,8 +366,22 @@ config TINY_RCU
is not required. This option greatly reduces the is not required. This option greatly reduces the
memory footprint of RCU. memory footprint of RCU.
config TINY_PREEMPT_RCU
bool "Preemptible UP-only small-memory-footprint RCU"
depends on !SMP && PREEMPT
help
This option selects the RCU implementation that is designed
for real-time UP systems. This option greatly reduces the
memory footprint of RCU.
endchoice endchoice
config PREEMPT_RCU
def_bool ( TREE_PREEMPT_RCU || TINY_PREEMPT_RCU )
help
This option enables preemptible-RCU code that is common between
the TREE_PREEMPT_RCU and TINY_PREEMPT_RCU implementations.
config RCU_TRACE config RCU_TRACE
bool "Enable tracing for RCU" bool "Enable tracing for RCU"
depends on TREE_RCU || TREE_PREEMPT_RCU depends on TREE_RCU || TREE_PREEMPT_RCU
...@@ -387,9 +402,12 @@ config RCU_FANOUT ...@@ -387,9 +402,12 @@ config RCU_FANOUT
help help
This option controls the fanout of hierarchical implementations This option controls the fanout of hierarchical implementations
of RCU, allowing RCU to work efficiently on machines with of RCU, allowing RCU to work efficiently on machines with
large numbers of CPUs. This value must be at least the cube large numbers of CPUs. This value must be at least the fourth
root of NR_CPUS, which allows NR_CPUS up to 32,768 for 32-bit root of NR_CPUS, which allows NR_CPUS to be insanely large.
systems and up to 262,144 for 64-bit systems. The default value of RCU_FANOUT should be used for production
systems, but if you are stress-testing the RCU implementation
itself, small RCU_FANOUT values allow you to test large-system
code paths on small(er) systems.
Select a specific number if testing RCU itself. Select a specific number if testing RCU itself.
Take the default if unsure. Take the default if unsure.
......
...@@ -86,6 +86,7 @@ obj-$(CONFIG_TREE_RCU) += rcutree.o ...@@ -86,6 +86,7 @@ obj-$(CONFIG_TREE_RCU) += rcutree.o
obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o
obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
obj-$(CONFIG_TINY_RCU) += rcutiny.o obj-$(CONFIG_TINY_RCU) += rcutiny.o
obj-$(CONFIG_TINY_PREEMPT_RCU) += rcutiny.o
obj-$(CONFIG_RELAY) += relay.o obj-$(CONFIG_RELAY) += relay.o
obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
......
...@@ -138,7 +138,7 @@ struct css_id { ...@@ -138,7 +138,7 @@ struct css_id {
* is called after synchronize_rcu(). But for safe use, css_is_removed() * is called after synchronize_rcu(). But for safe use, css_is_removed()
* css_tryget() should be used for avoiding race. * css_tryget() should be used for avoiding race.
*/ */
struct cgroup_subsys_state *css; struct cgroup_subsys_state __rcu *css;
/* /*
* ID of this css. * ID of this css.
*/ */
......
...@@ -401,7 +401,7 @@ struct task_struct *pid_task(struct pid *pid, enum pid_type type) ...@@ -401,7 +401,7 @@ struct task_struct *pid_task(struct pid *pid, enum pid_type type)
struct task_struct *result = NULL; struct task_struct *result = NULL;
if (pid) { if (pid) {
struct hlist_node *first; struct hlist_node *first;
first = rcu_dereference_check(pid->tasks[type].first, first = rcu_dereference_check(hlist_first_rcu(&pid->tasks[type]),
rcu_read_lock_held() || rcu_read_lock_held() ||
lockdep_tasklist_lock_is_held()); lockdep_tasklist_lock_is_held());
if (first) if (first)
...@@ -416,6 +416,7 @@ EXPORT_SYMBOL(pid_task); ...@@ -416,6 +416,7 @@ EXPORT_SYMBOL(pid_task);
*/ */
struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns) struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
{ {
rcu_lockdep_assert(rcu_read_lock_held());
return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID); return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
} }
......
...@@ -73,12 +73,14 @@ int debug_lockdep_rcu_enabled(void) ...@@ -73,12 +73,14 @@ int debug_lockdep_rcu_enabled(void)
EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled);
/** /**
* rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section? * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section?
* *
* Check for bottom half being disabled, which covers both the * Check for bottom half being disabled, which covers both the
* CONFIG_PROVE_RCU and not cases. Note that if someone uses * CONFIG_PROVE_RCU and not cases. Note that if someone uses
* rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled) * rcu_read_lock_bh(), but then later enables BH, lockdep (if enabled)
* will show the situation. * will show the situation. This is useful for debug checks in functions
* that require that they be called within an RCU read-side critical
* section.
* *
* Check debug_lockdep_rcu_enabled() to prevent false positives during boot. * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
*/ */
...@@ -86,7 +88,7 @@ int rcu_read_lock_bh_held(void) ...@@ -86,7 +88,7 @@ int rcu_read_lock_bh_held(void)
{ {
if (!debug_lockdep_rcu_enabled()) if (!debug_lockdep_rcu_enabled())
return 1; return 1;
return in_softirq(); return in_softirq() || irqs_disabled();
} }
EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
......
...@@ -59,6 +59,14 @@ int rcu_scheduler_active __read_mostly; ...@@ -59,6 +59,14 @@ int rcu_scheduler_active __read_mostly;
EXPORT_SYMBOL_GPL(rcu_scheduler_active); EXPORT_SYMBOL_GPL(rcu_scheduler_active);
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
/* Forward declarations for rcutiny_plugin.h. */
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp);
static void __call_rcu(struct rcu_head *head,
void (*func)(struct rcu_head *rcu),
struct rcu_ctrlblk *rcp);
#include "rcutiny_plugin.h"
#ifdef CONFIG_NO_HZ #ifdef CONFIG_NO_HZ
static long rcu_dynticks_nesting = 1; static long rcu_dynticks_nesting = 1;
...@@ -140,6 +148,7 @@ void rcu_check_callbacks(int cpu, int user) ...@@ -140,6 +148,7 @@ void rcu_check_callbacks(int cpu, int user)
rcu_sched_qs(cpu); rcu_sched_qs(cpu);
else if (!in_softirq()) else if (!in_softirq())
rcu_bh_qs(cpu); rcu_bh_qs(cpu);
rcu_preempt_check_callbacks();
} }
/* /*
...@@ -162,6 +171,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) ...@@ -162,6 +171,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
*rcp->donetail = NULL; *rcp->donetail = NULL;
if (rcp->curtail == rcp->donetail) if (rcp->curtail == rcp->donetail)
rcp->curtail = &rcp->rcucblist; rcp->curtail = &rcp->rcucblist;
rcu_preempt_remove_callbacks(rcp);
rcp->donetail = &rcp->rcucblist; rcp->donetail = &rcp->rcucblist;
local_irq_restore(flags); local_irq_restore(flags);
...@@ -182,6 +192,7 @@ static void rcu_process_callbacks(struct softirq_action *unused) ...@@ -182,6 +192,7 @@ static void rcu_process_callbacks(struct softirq_action *unused)
{ {
__rcu_process_callbacks(&rcu_sched_ctrlblk); __rcu_process_callbacks(&rcu_sched_ctrlblk);
__rcu_process_callbacks(&rcu_bh_ctrlblk); __rcu_process_callbacks(&rcu_bh_ctrlblk);
rcu_preempt_process_callbacks();
} }
/* /*
...@@ -223,15 +234,15 @@ static void __call_rcu(struct rcu_head *head, ...@@ -223,15 +234,15 @@ static void __call_rcu(struct rcu_head *head,
} }
/* /*
* Post an RCU callback to be invoked after the end of an RCU grace * Post an RCU callback to be invoked after the end of an RCU-sched grace
* period. But since we have but one CPU, that would be after any * period. But since we have but one CPU, that would be after any
* quiescent state. * quiescent state.
*/ */
void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
{ {
__call_rcu(head, func, &rcu_sched_ctrlblk); __call_rcu(head, func, &rcu_sched_ctrlblk);
} }
EXPORT_SYMBOL_GPL(call_rcu); EXPORT_SYMBOL_GPL(call_rcu_sched);
/* /*
* Post an RCU bottom-half callback to be invoked after any subsequent * Post an RCU bottom-half callback to be invoked after any subsequent
...@@ -243,20 +254,6 @@ void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) ...@@ -243,20 +254,6 @@ void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
} }
EXPORT_SYMBOL_GPL(call_rcu_bh); EXPORT_SYMBOL_GPL(call_rcu_bh);
void rcu_barrier(void)
{
struct rcu_synchronize rcu;
init_rcu_head_on_stack(&rcu.head);
init_completion(&rcu.completion);
/* Will wake me after RCU finished. */
call_rcu(&rcu.head, wakeme_after_rcu);
/* Wait for it. */
wait_for_completion(&rcu.completion);
destroy_rcu_head_on_stack(&rcu.head);
}
EXPORT_SYMBOL_GPL(rcu_barrier);
void rcu_barrier_bh(void) void rcu_barrier_bh(void)
{ {
struct rcu_synchronize rcu; struct rcu_synchronize rcu;
...@@ -289,5 +286,3 @@ void __init rcu_init(void) ...@@ -289,5 +286,3 @@ void __init rcu_init(void)
{ {
open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
} }
#include "rcutiny_plugin.h"
This diff is collapsed.
...@@ -120,7 +120,7 @@ struct rcu_torture { ...@@ -120,7 +120,7 @@ struct rcu_torture {
}; };
static LIST_HEAD(rcu_torture_freelist); static LIST_HEAD(rcu_torture_freelist);
static struct rcu_torture *rcu_torture_current; static struct rcu_torture __rcu *rcu_torture_current;
static long rcu_torture_current_version; static long rcu_torture_current_version;
static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
static DEFINE_SPINLOCK(rcu_torture_lock); static DEFINE_SPINLOCK(rcu_torture_lock);
...@@ -153,8 +153,10 @@ int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; ...@@ -153,8 +153,10 @@ int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
#define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */ #define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */
#define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */ #define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */
static int fullstop = FULLSTOP_RMMOD; static int fullstop = FULLSTOP_RMMOD;
DEFINE_MUTEX(fullstop_mutex); /* Protect fullstop transitions and spawning */ /*
/* of kthreads. */ * Protect fullstop transitions and spawning of kthreads.
*/
static DEFINE_MUTEX(fullstop_mutex);
/* /*
* Detect and respond to a system shutdown. * Detect and respond to a system shutdown.
...@@ -303,6 +305,10 @@ static void rcu_read_delay(struct rcu_random_state *rrsp) ...@@ -303,6 +305,10 @@ static void rcu_read_delay(struct rcu_random_state *rrsp)
mdelay(longdelay_ms); mdelay(longdelay_ms);
if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
udelay(shortdelay_us); udelay(shortdelay_us);
#ifdef CONFIG_PREEMPT
if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
preempt_schedule(); /* No QS if preempt_disable() in effect */
#endif
} }
static void rcu_torture_read_unlock(int idx) __releases(RCU) static void rcu_torture_read_unlock(int idx) __releases(RCU)
...@@ -536,6 +542,8 @@ static void srcu_read_delay(struct rcu_random_state *rrsp) ...@@ -536,6 +542,8 @@ static void srcu_read_delay(struct rcu_random_state *rrsp)
delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
if (!delay) if (!delay)
schedule_timeout_interruptible(longdelay); schedule_timeout_interruptible(longdelay);
else
rcu_read_delay(rrsp);
} }
static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
...@@ -731,7 +739,8 @@ rcu_torture_writer(void *arg) ...@@ -731,7 +739,8 @@ rcu_torture_writer(void *arg)
continue; continue;
rp->rtort_pipe_count = 0; rp->rtort_pipe_count = 0;
udelay(rcu_random(&rand) & 0x3ff); udelay(rcu_random(&rand) & 0x3ff);
old_rp = rcu_torture_current; old_rp = rcu_dereference_check(rcu_torture_current,
current == writer_task);
rp->rtort_mbtest = 1; rp->rtort_mbtest = 1;
rcu_assign_pointer(rcu_torture_current, rp); rcu_assign_pointer(rcu_torture_current, rp);
smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
......
...@@ -143,6 +143,11 @@ module_param(blimit, int, 0); ...@@ -143,6 +143,11 @@ module_param(blimit, int, 0);
module_param(qhimark, int, 0); module_param(qhimark, int, 0);
module_param(qlowmark, int, 0); module_param(qlowmark, int, 0);
#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
int rcu_cpu_stall_suppress __read_mostly = RCU_CPU_STALL_SUPPRESS_INIT;
module_param(rcu_cpu_stall_suppress, int, 0644);
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
static void force_quiescent_state(struct rcu_state *rsp, int relaxed); static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
static int rcu_pending(int cpu); static int rcu_pending(int cpu);
...@@ -450,7 +455,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) ...@@ -450,7 +455,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
#ifdef CONFIG_RCU_CPU_STALL_DETECTOR #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
int rcu_cpu_stall_panicking __read_mostly; int rcu_cpu_stall_suppress __read_mostly;
static void record_gp_stall_check_time(struct rcu_state *rsp) static void record_gp_stall_check_time(struct rcu_state *rsp)
{ {
...@@ -482,8 +487,11 @@ static void print_other_cpu_stall(struct rcu_state *rsp) ...@@ -482,8 +487,11 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
rcu_print_task_stall(rnp); rcu_print_task_stall(rnp);
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
/* OK, time to rat on our buddy... */ /*
* OK, time to rat on our buddy...
* See Documentation/RCU/stallwarn.txt for info on how to debug
* RCU CPU stall warnings.
*/
printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {", printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {",
rsp->name); rsp->name);
rcu_for_each_leaf_node(rsp, rnp) { rcu_for_each_leaf_node(rsp, rnp) {
...@@ -512,6 +520,11 @@ static void print_cpu_stall(struct rcu_state *rsp) ...@@ -512,6 +520,11 @@ static void print_cpu_stall(struct rcu_state *rsp)
unsigned long flags; unsigned long flags;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
/*
* OK, time to rat on ourselves...
* See Documentation/RCU/stallwarn.txt for info on how to debug
* RCU CPU stall warnings.
*/
printk(KERN_ERR "INFO: %s detected stall on CPU %d (t=%lu jiffies)\n", printk(KERN_ERR "INFO: %s detected stall on CPU %d (t=%lu jiffies)\n",
rsp->name, smp_processor_id(), jiffies - rsp->gp_start); rsp->name, smp_processor_id(), jiffies - rsp->gp_start);
trigger_all_cpu_backtrace(); trigger_all_cpu_backtrace();
...@@ -530,11 +543,11 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -530,11 +543,11 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
long delta; long delta;
struct rcu_node *rnp; struct rcu_node *rnp;
if (rcu_cpu_stall_panicking) if (rcu_cpu_stall_suppress)
return; return;
delta = jiffies - rsp->jiffies_stall; delta = jiffies - ACCESS_ONCE(rsp->jiffies_stall);
rnp = rdp->mynode; rnp = rdp->mynode;
if ((rnp->qsmask & rdp->grpmask) && delta >= 0) { if ((ACCESS_ONCE(rnp->qsmask) & rdp->grpmask) && delta >= 0) {
/* We haven't checked in, so go dump stack. */ /* We haven't checked in, so go dump stack. */
print_cpu_stall(rsp); print_cpu_stall(rsp);
...@@ -548,10 +561,26 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -548,10 +561,26 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr) static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
{ {
rcu_cpu_stall_panicking = 1; rcu_cpu_stall_suppress = 1;
return NOTIFY_DONE; return NOTIFY_DONE;
} }
/**
* rcu_cpu_stall_reset - prevent further stall warnings in current grace period
*
* Set the stall-warning timeout way off into the future, thus preventing
* any RCU CPU stall-warning messages from appearing in the current set of
* RCU grace periods.
*
* The caller must disable hard irqs.
*/
void rcu_cpu_stall_reset(void)
{
rcu_sched_state.jiffies_stall = jiffies + ULONG_MAX / 2;
rcu_bh_state.jiffies_stall = jiffies + ULONG_MAX / 2;
rcu_preempt_stall_reset();
}
static struct notifier_block rcu_panic_block = { static struct notifier_block rcu_panic_block = {
.notifier_call = rcu_panic, .notifier_call = rcu_panic,
}; };
...@@ -571,6 +600,10 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -571,6 +600,10 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
{ {
} }
void rcu_cpu_stall_reset(void)
{
}
static void __init check_cpu_stall_init(void) static void __init check_cpu_stall_init(void)
{ {
} }
...@@ -712,7 +745,7 @@ static void ...@@ -712,7 +745,7 @@ static void
rcu_start_gp(struct rcu_state *rsp, unsigned long flags) rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
__releases(rcu_get_root(rsp)->lock) __releases(rcu_get_root(rsp)->lock)
{ {
struct rcu_data *rdp = rsp->rda[smp_processor_id()]; struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) { if (!cpu_needs_another_gp(rsp, rdp) || rsp->fqs_active) {
...@@ -960,7 +993,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -960,7 +993,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp) static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
{ {
int i; int i;
struct rcu_data *rdp = rsp->rda[smp_processor_id()]; struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
if (rdp->nxtlist == NULL) if (rdp->nxtlist == NULL)
return; /* irqs disabled, so comparison is stable. */ return; /* irqs disabled, so comparison is stable. */
...@@ -971,6 +1004,7 @@ static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp) ...@@ -971,6 +1004,7 @@ static void rcu_send_cbs_to_orphanage(struct rcu_state *rsp)
for (i = 0; i < RCU_NEXT_SIZE; i++) for (i = 0; i < RCU_NEXT_SIZE; i++)
rdp->nxttail[i] = &rdp->nxtlist; rdp->nxttail[i] = &rdp->nxtlist;
rsp->orphan_qlen += rdp->qlen; rsp->orphan_qlen += rdp->qlen;
rdp->n_cbs_orphaned += rdp->qlen;
rdp->qlen = 0; rdp->qlen = 0;
raw_spin_unlock(&rsp->onofflock); /* irqs remain disabled. */ raw_spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
} }
...@@ -984,7 +1018,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp) ...@@ -984,7 +1018,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
struct rcu_data *rdp; struct rcu_data *rdp;
raw_spin_lock_irqsave(&rsp->onofflock, flags); raw_spin_lock_irqsave(&rsp->onofflock, flags);
rdp = rsp->rda[smp_processor_id()]; rdp = this_cpu_ptr(rsp->rda);
if (rsp->orphan_cbs_list == NULL) { if (rsp->orphan_cbs_list == NULL) {
raw_spin_unlock_irqrestore(&rsp->onofflock, flags); raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
return; return;
...@@ -992,6 +1026,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp) ...@@ -992,6 +1026,7 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp)
*rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_list; *rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_list;
rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_tail; rdp->nxttail[RCU_NEXT_TAIL] = rsp->orphan_cbs_tail;
rdp->qlen += rsp->orphan_qlen; rdp->qlen += rsp->orphan_qlen;
rdp->n_cbs_adopted += rsp->orphan_qlen;
rsp->orphan_cbs_list = NULL; rsp->orphan_cbs_list = NULL;
rsp->orphan_cbs_tail = &rsp->orphan_cbs_list; rsp->orphan_cbs_tail = &rsp->orphan_cbs_list;
rsp->orphan_qlen = 0; rsp->orphan_qlen = 0;
...@@ -1007,7 +1042,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) ...@@ -1007,7 +1042,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
unsigned long flags; unsigned long flags;
unsigned long mask; unsigned long mask;
int need_report = 0; int need_report = 0;
struct rcu_data *rdp = rsp->rda[cpu]; struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
struct rcu_node *rnp; struct rcu_node *rnp;
/* Exclude any attempts to start a new grace period. */ /* Exclude any attempts to start a new grace period. */
...@@ -1123,6 +1158,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) ...@@ -1123,6 +1158,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
/* Update count, and requeue any remaining callbacks. */ /* Update count, and requeue any remaining callbacks. */
rdp->qlen -= count; rdp->qlen -= count;
rdp->n_cbs_invoked += count;
if (list != NULL) { if (list != NULL) {
*tail = rdp->nxtlist; *tail = rdp->nxtlist;
rdp->nxtlist = list; rdp->nxtlist = list;
...@@ -1226,7 +1262,8 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *)) ...@@ -1226,7 +1262,8 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
cpu = rnp->grplo; cpu = rnp->grplo;
bit = 1; bit = 1;
for (; cpu <= rnp->grphi; cpu++, bit <<= 1) { for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu])) if ((rnp->qsmask & bit) != 0 &&
f(per_cpu_ptr(rsp->rda, cpu)))
mask |= bit; mask |= bit;
} }
if (mask != 0) { if (mask != 0) {
...@@ -1402,7 +1439,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), ...@@ -1402,7 +1439,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
* a quiescent state betweentimes. * a quiescent state betweentimes.
*/ */
local_irq_save(flags); local_irq_save(flags);
rdp = rsp->rda[smp_processor_id()]; rdp = this_cpu_ptr(rsp->rda);
rcu_process_gp_end(rsp, rdp); rcu_process_gp_end(rsp, rdp);
check_for_new_grace_period(rsp, rdp); check_for_new_grace_period(rsp, rdp);
...@@ -1701,7 +1738,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) ...@@ -1701,7 +1738,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
{ {
unsigned long flags; unsigned long flags;
int i; int i;
struct rcu_data *rdp = rsp->rda[cpu]; struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
/* Set up local state, ensuring consistent view of global state. */ /* Set up local state, ensuring consistent view of global state. */
...@@ -1729,7 +1766,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable) ...@@ -1729,7 +1766,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable)
{ {
unsigned long flags; unsigned long flags;
unsigned long mask; unsigned long mask;
struct rcu_data *rdp = rsp->rda[cpu]; struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
/* Set up local state, ensuring consistent view of global state. */ /* Set up local state, ensuring consistent view of global state. */
...@@ -1865,7 +1902,8 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp) ...@@ -1865,7 +1902,8 @@ static void __init rcu_init_levelspread(struct rcu_state *rsp)
/* /*
* Helper function for rcu_init() that initializes one rcu_state structure. * Helper function for rcu_init() that initializes one rcu_state structure.
*/ */
static void __init rcu_init_one(struct rcu_state *rsp) static void __init rcu_init_one(struct rcu_state *rsp,
struct rcu_data __percpu *rda)
{ {
static char *buf[] = { "rcu_node_level_0", static char *buf[] = { "rcu_node_level_0",
"rcu_node_level_1", "rcu_node_level_1",
...@@ -1918,37 +1956,23 @@ static void __init rcu_init_one(struct rcu_state *rsp) ...@@ -1918,37 +1956,23 @@ static void __init rcu_init_one(struct rcu_state *rsp)
} }
} }
rsp->rda = rda;
rnp = rsp->level[NUM_RCU_LVLS - 1]; rnp = rsp->level[NUM_RCU_LVLS - 1];
for_each_possible_cpu(i) { for_each_possible_cpu(i) {
while (i > rnp->grphi) while (i > rnp->grphi)
rnp++; rnp++;
rsp->rda[i]->mynode = rnp; per_cpu_ptr(rsp->rda, i)->mynode = rnp;
rcu_boot_init_percpu_data(i, rsp); rcu_boot_init_percpu_data(i, rsp);
} }
} }
/*
* Helper macro for __rcu_init() and __rcu_init_preempt(). To be used
* nowhere else! Assigns leaf node pointers into each CPU's rcu_data
* structure.
*/
#define RCU_INIT_FLAVOR(rsp, rcu_data) \
do { \
int i; \
\
for_each_possible_cpu(i) { \
(rsp)->rda[i] = &per_cpu(rcu_data, i); \
} \
rcu_init_one(rsp); \
} while (0)
void __init rcu_init(void) void __init rcu_init(void)
{ {
int cpu; int cpu;
rcu_bootup_announce(); rcu_bootup_announce();
RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data); rcu_init_one(&rcu_sched_state, &rcu_sched_data);
RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data); rcu_init_one(&rcu_bh_state, &rcu_bh_data);
__rcu_init_preempt(); __rcu_init_preempt();
open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
......
...@@ -202,6 +202,9 @@ struct rcu_data { ...@@ -202,6 +202,9 @@ struct rcu_data {
long qlen; /* # of queued callbacks */ long qlen; /* # of queued callbacks */
long qlen_last_fqs_check; long qlen_last_fqs_check;
/* qlen at last check for QS forcing */ /* qlen at last check for QS forcing */
unsigned long n_cbs_invoked; /* count of RCU cbs invoked. */
unsigned long n_cbs_orphaned; /* RCU cbs sent to orphanage. */
unsigned long n_cbs_adopted; /* RCU cbs adopted from orphanage. */
unsigned long n_force_qs_snap; unsigned long n_force_qs_snap;
/* did other CPU force QS recently? */ /* did other CPU force QS recently? */
long blimit; /* Upper limit on a processed batch */ long blimit; /* Upper limit on a processed batch */
...@@ -254,19 +257,23 @@ struct rcu_data { ...@@ -254,19 +257,23 @@ struct rcu_data {
#define RCU_STALL_DELAY_DELTA 0 #define RCU_STALL_DELAY_DELTA 0
#endif #endif
#define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ + RCU_STALL_DELAY_DELTA) #define RCU_SECONDS_TILL_STALL_CHECK (CONFIG_RCU_CPU_STALL_TIMEOUT * HZ + \
RCU_STALL_DELAY_DELTA)
/* for rsp->jiffies_stall */ /* for rsp->jiffies_stall */
#define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ + RCU_STALL_DELAY_DELTA) #define RCU_SECONDS_TILL_STALL_RECHECK (3 * RCU_SECONDS_TILL_STALL_CHECK + 30)
/* for rsp->jiffies_stall */ /* for rsp->jiffies_stall */
#define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */ #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */
/* to take at least one */ /* to take at least one */
/* scheduling clock irq */ /* scheduling clock irq */
/* before ratting on them. */ /* before ratting on them. */
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR_RUNNABLE
#define RCU_CPU_STALL_SUPPRESS_INIT 0
#else
#define RCU_CPU_STALL_SUPPRESS_INIT 1
#endif
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
/* /*
* RCU global state, including node hierarchy. This hierarchy is * RCU global state, including node hierarchy. This hierarchy is
...@@ -283,7 +290,7 @@ struct rcu_state { ...@@ -283,7 +290,7 @@ struct rcu_state {
struct rcu_node *level[NUM_RCU_LVLS]; /* Hierarchy levels. */ struct rcu_node *level[NUM_RCU_LVLS]; /* Hierarchy levels. */
u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */ u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */
u8 levelspread[NUM_RCU_LVLS]; /* kids/node in each level. */ u8 levelspread[NUM_RCU_LVLS]; /* kids/node in each level. */
struct rcu_data *rda[NR_CPUS]; /* array of rdp pointers. */ struct rcu_data __percpu *rda; /* pointer of percu rcu_data. */
/* The following fields are guarded by the root rcu_node's lock. */ /* The following fields are guarded by the root rcu_node's lock. */
...@@ -365,6 +372,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, ...@@ -365,6 +372,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
#ifdef CONFIG_RCU_CPU_STALL_DETECTOR #ifdef CONFIG_RCU_CPU_STALL_DETECTOR
static void rcu_print_detail_task_stall(struct rcu_state *rsp); static void rcu_print_detail_task_stall(struct rcu_state *rsp);
static void rcu_print_task_stall(struct rcu_node *rnp); static void rcu_print_task_stall(struct rcu_node *rnp);
static void rcu_preempt_stall_reset(void);
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU
......
...@@ -57,7 +57,7 @@ static void __init rcu_bootup_announce_oddness(void) ...@@ -57,7 +57,7 @@ static void __init rcu_bootup_announce_oddness(void)
printk(KERN_INFO printk(KERN_INFO
"\tRCU-based detection of stalled CPUs is disabled.\n"); "\tRCU-based detection of stalled CPUs is disabled.\n");
#endif #endif
#ifndef CONFIG_RCU_CPU_STALL_VERBOSE #if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n"); printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n");
#endif #endif
#if NUM_RCU_LVL_4 != 0 #if NUM_RCU_LVL_4 != 0
...@@ -154,7 +154,7 @@ static void rcu_preempt_note_context_switch(int cpu) ...@@ -154,7 +154,7 @@ static void rcu_preempt_note_context_switch(int cpu)
(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) { (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
/* Possibly blocking in an RCU read-side critical section. */ /* Possibly blocking in an RCU read-side critical section. */
rdp = rcu_preempt_state.rda[cpu]; rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED; t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
...@@ -201,7 +201,7 @@ static void rcu_preempt_note_context_switch(int cpu) ...@@ -201,7 +201,7 @@ static void rcu_preempt_note_context_switch(int cpu)
*/ */
void __rcu_read_lock(void) void __rcu_read_lock(void)
{ {
ACCESS_ONCE(current->rcu_read_lock_nesting)++; current->rcu_read_lock_nesting++;
barrier(); /* needed if we ever invoke rcu_read_lock in rcutree.c */ barrier(); /* needed if we ever invoke rcu_read_lock in rcutree.c */
} }
EXPORT_SYMBOL_GPL(__rcu_read_lock); EXPORT_SYMBOL_GPL(__rcu_read_lock);
...@@ -344,7 +344,9 @@ void __rcu_read_unlock(void) ...@@ -344,7 +344,9 @@ void __rcu_read_unlock(void)
struct task_struct *t = current; struct task_struct *t = current;
barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */ barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
if (--ACCESS_ONCE(t->rcu_read_lock_nesting) == 0 && --t->rcu_read_lock_nesting;
barrier(); /* decrement before load of ->rcu_read_unlock_special */
if (t->rcu_read_lock_nesting == 0 &&
unlikely(ACCESS_ONCE(t->rcu_read_unlock_special))) unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
rcu_read_unlock_special(t); rcu_read_unlock_special(t);
#ifdef CONFIG_PROVE_LOCKING #ifdef CONFIG_PROVE_LOCKING
...@@ -417,6 +419,16 @@ static void rcu_print_task_stall(struct rcu_node *rnp) ...@@ -417,6 +419,16 @@ static void rcu_print_task_stall(struct rcu_node *rnp)
} }
} }
/*
* Suppress preemptible RCU's CPU stall warnings by pushing the
* time of the next stall-warning message comfortably far into the
* future.
*/
static void rcu_preempt_stall_reset(void)
{
rcu_preempt_state.jiffies_stall = jiffies + ULONG_MAX / 2;
}
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
/* /*
...@@ -546,9 +558,11 @@ EXPORT_SYMBOL_GPL(call_rcu); ...@@ -546,9 +558,11 @@ EXPORT_SYMBOL_GPL(call_rcu);
* *
* Control will return to the caller some time after a full grace * Control will return to the caller some time after a full grace
* period has elapsed, in other words after all currently executing RCU * period has elapsed, in other words after all currently executing RCU
* read-side critical sections have completed. RCU read-side critical * read-side critical sections have completed. Note, however, that
* sections are delimited by rcu_read_lock() and rcu_read_unlock(), * upon return from synchronize_rcu(), the caller might well be executing
* and may be nested. * concurrently with new RCU read-side critical sections that began while
* synchronize_rcu() was waiting. RCU read-side critical sections are
* delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
*/ */
void synchronize_rcu(void) void synchronize_rcu(void)
{ {
...@@ -771,7 +785,7 @@ static void rcu_preempt_send_cbs_to_orphanage(void) ...@@ -771,7 +785,7 @@ static void rcu_preempt_send_cbs_to_orphanage(void)
*/ */
static void __init __rcu_init_preempt(void) static void __init __rcu_init_preempt(void)
{ {
RCU_INIT_FLAVOR(&rcu_preempt_state, rcu_preempt_data); rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
} }
/* /*
...@@ -865,6 +879,14 @@ static void rcu_print_task_stall(struct rcu_node *rnp) ...@@ -865,6 +879,14 @@ static void rcu_print_task_stall(struct rcu_node *rnp)
{ {
} }
/*
* Because preemptible RCU does not exist, there is no need to suppress
* its CPU stall warnings.
*/
static void rcu_preempt_stall_reset(void)
{
}
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
/* /*
...@@ -918,15 +940,6 @@ static void rcu_preempt_process_callbacks(void) ...@@ -918,15 +940,6 @@ static void rcu_preempt_process_callbacks(void)
{ {
} }
/*
* In classic RCU, call_rcu() is just call_rcu_sched().
*/
void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
{
call_rcu_sched(head, func);
}
EXPORT_SYMBOL_GPL(call_rcu);
/* /*
* Wait for an rcu-preempt grace period, but make it happen quickly. * Wait for an rcu-preempt grace period, but make it happen quickly.
* But because preemptable RCU does not exist, map to rcu-sched. * But because preemptable RCU does not exist, map to rcu-sched.
......
...@@ -64,7 +64,9 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) ...@@ -64,7 +64,9 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
rdp->dynticks_fqs); rdp->dynticks_fqs);
#endif /* #ifdef CONFIG_NO_HZ */ #endif /* #ifdef CONFIG_NO_HZ */
seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi); seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
seq_printf(m, " ql=%ld b=%ld\n", rdp->qlen, rdp->blimit); seq_printf(m, " ql=%ld b=%ld", rdp->qlen, rdp->blimit);
seq_printf(m, " ci=%lu co=%lu ca=%lu\n",
rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
} }
#define PRINT_RCU_DATA(name, func, m) \ #define PRINT_RCU_DATA(name, func, m) \
...@@ -119,7 +121,9 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp) ...@@ -119,7 +121,9 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
rdp->dynticks_fqs); rdp->dynticks_fqs);
#endif /* #ifdef CONFIG_NO_HZ */ #endif /* #ifdef CONFIG_NO_HZ */
seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi); seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
seq_printf(m, ",%ld,%ld\n", rdp->qlen, rdp->blimit); seq_printf(m, ",%ld,%ld", rdp->qlen, rdp->blimit);
seq_printf(m, ",%lu,%lu,%lu\n",
rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
} }
static int show_rcudata_csv(struct seq_file *m, void *unused) static int show_rcudata_csv(struct seq_file *m, void *unused)
...@@ -128,7 +132,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused) ...@@ -128,7 +132,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused)
#ifdef CONFIG_NO_HZ #ifdef CONFIG_NO_HZ
seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\","); seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
#endif /* #ifdef CONFIG_NO_HZ */ #endif /* #ifdef CONFIG_NO_HZ */
seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n"); seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n");
#ifdef CONFIG_TREE_PREEMPT_RCU #ifdef CONFIG_TREE_PREEMPT_RCU
seq_puts(m, "\"rcu_preempt:\"\n"); seq_puts(m, "\"rcu_preempt:\"\n");
PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m); PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);
...@@ -262,7 +266,7 @@ static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp) ...@@ -262,7 +266,7 @@ static void print_rcu_pendings(struct seq_file *m, struct rcu_state *rsp)
struct rcu_data *rdp; struct rcu_data *rdp;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
rdp = rsp->rda[cpu]; rdp = per_cpu_ptr(rsp->rda, cpu);
if (rdp->beenonline) if (rdp->beenonline)
print_one_rcu_pending(m, rdp); print_one_rcu_pending(m, rdp);
} }
......
...@@ -5337,7 +5337,19 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) ...@@ -5337,7 +5337,19 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
idle->se.exec_start = sched_clock(); idle->se.exec_start = sched_clock();
cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu)); cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
/*
* We're having a chicken and egg problem, even though we are
* holding rq->lock, the cpu isn't yet set to this cpu so the
* lockdep check in task_group() will fail.
*
* Similar case to sched_fork(). / Alternatively we could
* use task_rq_lock() here and obtain the other rq->lock.
*
* Silence PROVE_RCU
*/
rcu_read_lock();
__set_task_cpu(idle, cpu); __set_task_cpu(idle, cpu);
rcu_read_unlock();
rq->curr = rq->idle = idle; rq->curr = rq->idle = idle;
#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
......
...@@ -3751,8 +3751,11 @@ static void task_fork_fair(struct task_struct *p) ...@@ -3751,8 +3751,11 @@ static void task_fork_fair(struct task_struct *p)
update_rq_clock(rq); update_rq_clock(rq);
if (unlikely(task_cpu(p) != this_cpu)) if (unlikely(task_cpu(p) != this_cpu)) {
rcu_read_lock();
__set_task_cpu(p, this_cpu); __set_task_cpu(p, this_cpu);
rcu_read_unlock();
}
update_curr(cfs_rq); update_curr(cfs_rq);
......
...@@ -46,11 +46,9 @@ static int init_srcu_struct_fields(struct srcu_struct *sp) ...@@ -46,11 +46,9 @@ static int init_srcu_struct_fields(struct srcu_struct *sp)
int __init_srcu_struct(struct srcu_struct *sp, const char *name, int __init_srcu_struct(struct srcu_struct *sp, const char *name,
struct lock_class_key *key) struct lock_class_key *key)
{ {
#ifdef CONFIG_DEBUG_LOCK_ALLOC
/* Don't re-initialize a lock while it is held. */ /* Don't re-initialize a lock while it is held. */
debug_check_no_locks_freed((void *)sp, sizeof(*sp)); debug_check_no_locks_freed((void *)sp, sizeof(*sp));
lockdep_init_map(&sp->dep_map, name, key, 0); lockdep_init_map(&sp->dep_map, name, key, 0);
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
return init_srcu_struct_fields(sp); return init_srcu_struct_fields(sp);
} }
EXPORT_SYMBOL_GPL(__init_srcu_struct); EXPORT_SYMBOL_GPL(__init_srcu_struct);
......
...@@ -539,6 +539,23 @@ config PROVE_RCU_REPEATEDLY ...@@ -539,6 +539,23 @@ config PROVE_RCU_REPEATEDLY
disabling, allowing multiple RCU-lockdep warnings to be printed disabling, allowing multiple RCU-lockdep warnings to be printed
on a single reboot. on a single reboot.
Say Y to allow multiple RCU-lockdep warnings per boot.
Say N if you are unsure.
config SPARSE_RCU_POINTER
bool "RCU debugging: sparse-based checks for pointer usage"
default n
help
This feature enables the __rcu sparse annotation for
RCU-protected pointers. This annotation will cause sparse
to flag any non-RCU used of annotated pointers. This can be
helpful when debugging RCU usage. Please note that this feature
is not intended to enforce code cleanliness; it is instead merely
a debugging aid.
Say Y to make sparse flag questionable use of RCU-protected pointers
Say N if you are unsure. Say N if you are unsure.
config LOCKDEP config LOCKDEP
...@@ -832,6 +849,30 @@ config RCU_CPU_STALL_DETECTOR ...@@ -832,6 +849,30 @@ config RCU_CPU_STALL_DETECTOR
Say Y if you are unsure. Say Y if you are unsure.
config RCU_CPU_STALL_TIMEOUT
int "RCU CPU stall timeout in seconds"
depends on RCU_CPU_STALL_DETECTOR
range 3 300
default 60
help
If a given RCU grace period extends more than the specified
number of seconds, a CPU stall warning is printed. If the
RCU grace period persists, additional CPU stall warnings are
printed at more widely spaced intervals.
config RCU_CPU_STALL_DETECTOR_RUNNABLE
bool "RCU CPU stall checking starts automatically at boot"
depends on RCU_CPU_STALL_DETECTOR
default y
help
If set, start checking for RCU CPU stalls immediately on
boot. Otherwise, RCU CPU stall checking must be manually
enabled.
Say Y if you are unsure.
Say N if you wish to suppress RCU CPU stall checking during boot.
config RCU_CPU_STALL_VERBOSE config RCU_CPU_STALL_VERBOSE
bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR" bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR"
depends on RCU_CPU_STALL_DETECTOR && TREE_PREEMPT_RCU depends on RCU_CPU_STALL_DETECTOR && TREE_PREEMPT_RCU
......
...@@ -49,7 +49,7 @@ struct radix_tree_node { ...@@ -49,7 +49,7 @@ struct radix_tree_node {
unsigned int height; /* Height from the bottom */ unsigned int height; /* Height from the bottom */
unsigned int count; unsigned int count;
struct rcu_head rcu_head; struct rcu_head rcu_head;
void *slots[RADIX_TREE_MAP_SIZE]; void __rcu *slots[RADIX_TREE_MAP_SIZE];
unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS]; unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
}; };
......
...@@ -1078,8 +1078,11 @@ static void sk_prot_free(struct proto *prot, struct sock *sk) ...@@ -1078,8 +1078,11 @@ static void sk_prot_free(struct proto *prot, struct sock *sk)
#ifdef CONFIG_CGROUPS #ifdef CONFIG_CGROUPS
void sock_update_classid(struct sock *sk) void sock_update_classid(struct sock *sk)
{ {
u32 classid = task_cls_classid(current); u32 classid;
rcu_read_lock(); /* doing current task, which cannot vanish. */
classid = task_cls_classid(current);
rcu_read_unlock();
if (classid && classid != sk->sk_classid) if (classid && classid != sk->sk_classid)
sk->sk_classid = classid; sk->sk_classid = classid;
} }
......
...@@ -38,7 +38,7 @@ static DEFINE_SPINLOCK(nf_nat_lock); ...@@ -38,7 +38,7 @@ static DEFINE_SPINLOCK(nf_nat_lock);
static struct nf_conntrack_l3proto *l3proto __read_mostly; static struct nf_conntrack_l3proto *l3proto __read_mostly;
#define MAX_IP_NAT_PROTO 256 #define MAX_IP_NAT_PROTO 256
static const struct nf_nat_protocol *nf_nat_protos[MAX_IP_NAT_PROTO] static const struct nf_nat_protocol __rcu *nf_nat_protos[MAX_IP_NAT_PROTO]
__read_mostly; __read_mostly;
static inline const struct nf_nat_protocol * static inline const struct nf_nat_protocol *
......
...@@ -27,7 +27,7 @@ ...@@ -27,7 +27,7 @@
static DEFINE_MUTEX(afinfo_mutex); static DEFINE_MUTEX(afinfo_mutex);
const struct nf_afinfo *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly; const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
EXPORT_SYMBOL(nf_afinfo); EXPORT_SYMBOL(nf_afinfo);
int nf_register_afinfo(const struct nf_afinfo *afinfo) int nf_register_afinfo(const struct nf_afinfo *afinfo)
......
...@@ -26,10 +26,10 @@ ...@@ -26,10 +26,10 @@
static DEFINE_MUTEX(nf_ct_ecache_mutex); static DEFINE_MUTEX(nf_ct_ecache_mutex);
struct nf_ct_event_notifier *nf_conntrack_event_cb __read_mostly; struct nf_ct_event_notifier __rcu *nf_conntrack_event_cb __read_mostly;
EXPORT_SYMBOL_GPL(nf_conntrack_event_cb); EXPORT_SYMBOL_GPL(nf_conntrack_event_cb);
struct nf_exp_event_notifier *nf_expect_event_cb __read_mostly; struct nf_exp_event_notifier __rcu *nf_expect_event_cb __read_mostly;
EXPORT_SYMBOL_GPL(nf_expect_event_cb); EXPORT_SYMBOL_GPL(nf_expect_event_cb);
/* deliver cached events and clear cache entry - must be called with locally /* deliver cached events and clear cache entry - must be called with locally
......
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
#include <linux/skbuff.h> #include <linux/skbuff.h>
#include <net/netfilter/nf_conntrack_extend.h> #include <net/netfilter/nf_conntrack_extend.h>
static struct nf_ct_ext_type *nf_ct_ext_types[NF_CT_EXT_NUM]; static struct nf_ct_ext_type __rcu *nf_ct_ext_types[NF_CT_EXT_NUM];
static DEFINE_MUTEX(nf_ct_ext_type_mutex); static DEFINE_MUTEX(nf_ct_ext_type_mutex);
void __nf_ct_ext_destroy(struct nf_conn *ct) void __nf_ct_ext_destroy(struct nf_conn *ct)
......
...@@ -28,8 +28,8 @@ ...@@ -28,8 +28,8 @@
#include <net/netfilter/nf_conntrack_l4proto.h> #include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_core.h> #include <net/netfilter/nf_conntrack_core.h>
static struct nf_conntrack_l4proto **nf_ct_protos[PF_MAX] __read_mostly; static struct nf_conntrack_l4proto __rcu **nf_ct_protos[PF_MAX] __read_mostly;
struct nf_conntrack_l3proto *nf_ct_l3protos[AF_MAX] __read_mostly; struct nf_conntrack_l3proto __rcu *nf_ct_l3protos[AF_MAX] __read_mostly;
EXPORT_SYMBOL_GPL(nf_ct_l3protos); EXPORT_SYMBOL_GPL(nf_ct_l3protos);
static DEFINE_MUTEX(nf_ct_proto_mutex); static DEFINE_MUTEX(nf_ct_proto_mutex);
......
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
#define NF_LOG_PREFIXLEN 128 #define NF_LOG_PREFIXLEN 128
#define NFLOGGER_NAME_LEN 64 #define NFLOGGER_NAME_LEN 64
static const struct nf_logger *nf_loggers[NFPROTO_NUMPROTO] __read_mostly; static const struct nf_logger __rcu *nf_loggers[NFPROTO_NUMPROTO] __read_mostly;
static struct list_head nf_loggers_l[NFPROTO_NUMPROTO] __read_mostly; static struct list_head nf_loggers_l[NFPROTO_NUMPROTO] __read_mostly;
static DEFINE_MUTEX(nf_log_mutex); static DEFINE_MUTEX(nf_log_mutex);
......
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
* long term mutex. The handler must provide an an outfn() to accept packets * long term mutex. The handler must provide an an outfn() to accept packets
* for queueing and must reinject all packets it receives, no matter what. * for queueing and must reinject all packets it receives, no matter what.
*/ */
static const struct nf_queue_handler *queue_handler[NFPROTO_NUMPROTO] __read_mostly; static const struct nf_queue_handler __rcu *queue_handler[NFPROTO_NUMPROTO] __read_mostly;
static DEFINE_MUTEX(queue_handler_mutex); static DEFINE_MUTEX(queue_handler_mutex);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment