Skip to content

L
linux
Commit ba6306e3 authored by Herbert Xu's avatar Herbert Xu Committed by David S. Miller
Browse files
Options

rhashtable: Remove RCU marking from rhash_lock_head 

The opaque type rhash_lock_head should not be marked with __rcu
because it can never be dereferenced.  We should apply the RCU
marking when we turn it into a pointer which can be dereferenced.

This patch does exactly that.  This fixes a number of sparse
warnings as well as getting rid of some unnecessary RCU checking.
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent c7d5ec26
Show whitespace changes
Inline Side-by-side
Showing with 46 additions and 40 deletions
include/linux/rhashtable.h
View file @ ba6306e3
...@@ -84,7 +84,7 @@ struct bucket_table { ...@@ -84,7 +84,7 @@ struct bucket_table {
struct lockdep_map dep_map; struct lockdep_map dep_map;
struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp; struct rhash_lock_head *buckets[] ____cacheline_aligned_in_smp;
}; };
/* /*
...@@ -261,11 +261,11 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, ...@@ -261,11 +261,11 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
void *arg); void *arg);
void rhashtable_destroy(struct rhashtable *ht); void rhashtable_destroy(struct rhashtable *ht);
struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash); unsigned int hash);
struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash); unsigned int hash);
struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
struct bucket_table *tbl, struct bucket_table *tbl,
unsigned int hash); unsigned int hash);
...@@ -284,21 +284,21 @@ struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, ...@@ -284,21 +284,21 @@ struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
#define rht_entry(tpos, pos, member) \ #define rht_entry(tpos, pos, member) \
({ tpos = container_of(pos, typeof(*tpos), member); 1; }) ({ tpos = container_of(pos, typeof(*tpos), member); 1; })
static inline struct rhash_lock_head __rcu *const *rht_bucket( static inline struct rhash_lock_head *const *rht_bucket(
const struct bucket_table *tbl, unsigned int hash) const struct bucket_table *tbl, unsigned int hash)
{ {
return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash]; &tbl->buckets[hash];
} }
static inline struct rhash_lock_head __rcu **rht_bucket_var( static inline struct rhash_lock_head **rht_bucket_var(
struct bucket_table *tbl, unsigned int hash) struct bucket_table *tbl, unsigned int hash)
{ {
return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) : return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash]; &tbl->buckets[hash];
} }
static inline struct rhash_lock_head __rcu **rht_bucket_insert( static inline struct rhash_lock_head **rht_bucket_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash) struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
{ {
return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) : return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
...@@ -349,6 +349,12 @@ static inline void rht_unlock(struct bucket_table *tbl, ...@@ -349,6 +349,12 @@ static inline void rht_unlock(struct bucket_table *tbl,
local_bh_enable(); local_bh_enable();
} }
static inline struct rhash_head __rcu *__rht_ptr(
struct rhash_lock_head *const *bkt)
{
return (struct rhash_head __rcu *)((unsigned long)*bkt & ~BIT(0));
}
/* /*
* Where 'bkt' is a bucket and might be locked: * Where 'bkt' is a bucket and might be locked:
* rht_ptr() dereferences that pointer and clears the lock bit. * rht_ptr() dereferences that pointer and clears the lock bit.
...@@ -356,30 +362,30 @@ static inline void rht_unlock(struct bucket_table *tbl, ...@@ -356,30 +362,30 @@ static inline void rht_unlock(struct bucket_table *tbl,
* access is guaranteed, such as when destroying the table. * access is guaranteed, such as when destroying the table.
*/ */
static inline struct rhash_head *rht_ptr( static inline struct rhash_head *rht_ptr(
struct rhash_lock_head __rcu * const *bkt, struct rhash_lock_head *const *bkt,
struct bucket_table *tbl, struct bucket_table *tbl,
unsigned int hash) unsigned int hash)
{ {
const struct rhash_lock_head *p = struct rhash_head __rcu *p = __rht_ptr(bkt);
rht_dereference_bucket_rcu(*bkt, tbl, hash);
if ((((unsigned long)p) & ~BIT(0)) == 0) if (!p)
return RHT_NULLS_MARKER(bkt); return RHT_NULLS_MARKER(bkt);
return (void *)(((unsigned long)p) & ~BIT(0));
return rht_dereference_bucket_rcu(p, tbl, hash);
} }
static inline struct rhash_head *rht_ptr_exclusive( static inline struct rhash_head *rht_ptr_exclusive(
struct rhash_lock_head __rcu * const *bkt) struct rhash_lock_head *const *bkt)
{ {
const struct rhash_lock_head *p = struct rhash_head __rcu *p = __rht_ptr(bkt);
rcu_dereference_protected(*bkt, 1);
if (!p) if (!p)
return RHT_NULLS_MARKER(bkt); return RHT_NULLS_MARKER(bkt);
return (void *)(((unsigned long)p) & ~BIT(0));
return rcu_dereference_protected(p, 1);
} }
static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, static inline void rht_assign_locked(struct rhash_lock_head **bkt,
struct rhash_head *obj) struct rhash_head *obj)
{ {
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt; struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
...@@ -390,7 +396,7 @@ static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, ...@@ -390,7 +396,7 @@ static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
} }
static inline void rht_assign_unlock(struct bucket_table *tbl, static inline void rht_assign_unlock(struct bucket_table *tbl,
struct rhash_lock_head __rcu **bkt, struct rhash_lock_head **bkt,
struct rhash_head *obj) struct rhash_head *obj)
{ {
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt; struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
...@@ -587,7 +593,7 @@ static inline struct rhash_head *__rhashtable_lookup( ...@@ -587,7 +593,7 @@ static inline struct rhash_head *__rhashtable_lookup(
.ht = ht, .ht = ht,
.key = key, .key = key,
}; };
struct rhash_lock_head __rcu * const *bkt; struct rhash_lock_head *const *bkt;
struct bucket_table *tbl; struct bucket_table *tbl;
struct rhash_head *he; struct rhash_head *he;
unsigned int hash; unsigned int hash;
...@@ -703,7 +709,7 @@ static inline void *__rhashtable_insert_fast( ...@@ -703,7 +709,7 @@ static inline void *__rhashtable_insert_fast(
.ht = ht, .ht = ht,
.key = key, .key = key,
}; };
struct rhash_lock_head __rcu **bkt; struct rhash_lock_head **bkt;
struct rhash_head __rcu **pprev; struct rhash_head __rcu **pprev;
struct bucket_table *tbl; struct bucket_table *tbl;
struct rhash_head *head; struct rhash_head *head;
...@@ -989,7 +995,7 @@ static inline int __rhashtable_remove_fast_one( ...@@ -989,7 +995,7 @@ static inline int __rhashtable_remove_fast_one(
struct rhash_head *obj, const struct rhashtable_params params, struct rhash_head *obj, const struct rhashtable_params params,
bool rhlist) bool rhlist)
{ {
struct rhash_lock_head __rcu **bkt; struct rhash_lock_head **bkt;
struct rhash_head __rcu **pprev; struct rhash_head __rcu **pprev;
struct rhash_head *he; struct rhash_head *he;
unsigned int hash; unsigned int hash;
...@@ -1141,7 +1147,7 @@ static inline int __rhashtable_replace_fast( ...@@ -1141,7 +1147,7 @@ static inline int __rhashtable_replace_fast(
struct rhash_head *obj_old, struct rhash_head *obj_new, struct rhash_head *obj_old, struct rhash_head *obj_new,
const struct rhashtable_params params) const struct rhashtable_params params)
{ {
struct rhash_lock_head __rcu **bkt; struct rhash_lock_head **bkt;
struct rhash_head __rcu **pprev; struct rhash_head __rcu **pprev;
struct rhash_head *he; struct rhash_head *he;
unsigned int hash; unsigned int hash;
......
lib/rhashtable.c
View file @ ba6306e3
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
union nested_table { union nested_table {
union nested_table __rcu *table; union nested_table __rcu *table;
struct rhash_lock_head __rcu *bucket; struct rhash_lock_head *bucket;
}; };
static u32 head_hashfn(struct rhashtable *ht, static u32 head_hashfn(struct rhashtable *ht,
...@@ -216,7 +216,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, ...@@ -216,7 +216,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht,
} }
static int rhashtable_rehash_one(struct rhashtable *ht, static int rhashtable_rehash_one(struct rhashtable *ht,
struct rhash_lock_head __rcu **bkt, struct rhash_lock_head **bkt,
unsigned int old_hash) unsigned int old_hash)
{ {
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
...@@ -269,7 +269,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, ...@@ -269,7 +269,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht,
unsigned int old_hash) unsigned int old_hash)
{ {
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash); struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash);
int err; int err;
if (!bkt) if (!bkt)
...@@ -478,7 +478,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, ...@@ -478,7 +478,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht,
} }
static void *rhashtable_lookup_one(struct rhashtable *ht, static void *rhashtable_lookup_one(struct rhashtable *ht,
struct rhash_lock_head __rcu **bkt, struct rhash_lock_head **bkt,
struct bucket_table *tbl, unsigned int hash, struct bucket_table *tbl, unsigned int hash,
const void *key, struct rhash_head *obj) const void *key, struct rhash_head *obj)
{ {
...@@ -529,7 +529,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, ...@@ -529,7 +529,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht,
} }
static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
struct rhash_lock_head __rcu **bkt, struct rhash_lock_head **bkt,
struct bucket_table *tbl, struct bucket_table *tbl,
unsigned int hash, unsigned int hash,
struct rhash_head *obj, struct rhash_head *obj,
...@@ -584,7 +584,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, ...@@ -584,7 +584,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
{ {
struct bucket_table *new_tbl; struct bucket_table *new_tbl;
struct bucket_table *tbl; struct bucket_table *tbl;
struct rhash_lock_head __rcu **bkt; struct rhash_lock_head **bkt;
unsigned int hash; unsigned int hash;
void *data; void *data;
...@@ -1166,7 +1166,7 @@ void rhashtable_destroy(struct rhashtable *ht) ...@@ -1166,7 +1166,7 @@ void rhashtable_destroy(struct rhashtable *ht)
} }
EXPORT_SYMBOL_GPL(rhashtable_destroy); EXPORT_SYMBOL_GPL(rhashtable_destroy);
struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash) unsigned int hash)
{ {
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
...@@ -1195,10 +1195,10 @@ struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tb ...@@ -1195,10 +1195,10 @@ struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tb
} }
EXPORT_SYMBOL_GPL(__rht_bucket_nested); EXPORT_SYMBOL_GPL(__rht_bucket_nested);
struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash) unsigned int hash)
{ {
static struct rhash_lock_head __rcu *rhnull; static struct rhash_lock_head *rhnull;
if (!rhnull) if (!rhnull)
INIT_RHT_NULLS_HEAD(rhnull); INIT_RHT_NULLS_HEAD(rhnull);
...@@ -1206,7 +1206,7 @@ struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, ...@@ -1206,7 +1206,7 @@ struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
} }
EXPORT_SYMBOL_GPL(rht_bucket_nested); EXPORT_SYMBOL_GPL(rht_bucket_nested);
struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
struct bucket_table *tbl, struct bucket_table *tbl,
unsigned int hash) unsigned int hash)
{ {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7