Skip to content

L
linux
Commit 44b0c295 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files
Options

u64_stats: Streamline the implementation 

The u64 stats code handles 3 different cases:

  - 32bit UP
  - 32bit SMP
  - 64bit

with an unreadable #ifdef maze, which was recently expanded with PREEMPT_RT
conditionals.

Reduce it to two cases (32bit and 64bit) and drop the optimization for
32bit UP as suggested by Linus.

Use the new preempt_disable/enable_nested() helpers to get rid of the
CONFIG_PREEMPT_RT conditionals.
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220825164131.402717-9-bigeasy@linutronix.de
parent 9458e0a7
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 81 deletions
include/linux/u64_stats_sync.h
View file @ 44b0c295
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
* *
* Key points : * Key points :
* *
* - Use a seqcount on 32-bit SMP, only disable preemption for 32-bit UP. * - Use a seqcount on 32-bit
* - The whole thing is a no-op on 64-bit architectures. * - The whole thing is a no-op on 64-bit architectures.
* *
* Usage constraints: * Usage constraints:
...@@ -20,7 +20,8 @@ ...@@ -20,7 +20,8 @@
* writer and also spin forever. * writer and also spin forever.
* *
* 3) Write side must use the _irqsave() variant if other writers, or a reader, * 3) Write side must use the _irqsave() variant if other writers, or a reader,
* can be invoked from an IRQ context. * can be invoked from an IRQ context. On 64bit systems this variant does not
* disable interrupts.
* *
* 4) If reader fetches several counters, there is no guarantee the whole values * 4) If reader fetches several counters, there is no guarantee the whole values
* are consistent w.r.t. each other (remember point #2: seqcounts are not * are consistent w.r.t. each other (remember point #2: seqcounts are not
...@@ -29,11 +30,6 @@ ...@@ -29,11 +30,6 @@
* 5) Readers are allowed to sleep or be preempted/interrupted: they perform * 5) Readers are allowed to sleep or be preempted/interrupted: they perform
* pure reads. * pure reads.
* *
* 6) Readers must use both u64_stats_fetch_{begin,retry}_irq() if the stats
* might be updated from a hardirq or softirq context (remember point #1:
* seqcounts are not used for UP kernels). 32-bit UP stat readers could read
* corrupted 64-bit values otherwise.
*
* Usage : * Usage :
* *
* Stats producer (writer) should use following template granted it already got * Stats producer (writer) should use following template granted it already got
...@@ -66,7 +62,7 @@ ...@@ -66,7 +62,7 @@
#include <linux/seqlock.h> #include <linux/seqlock.h>
struct u64_stats_sync { struct u64_stats_sync {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) #if BITS_PER_LONG == 32
seqcount_t seq; seqcount_t seq;
#endif #endif
}; };
...@@ -98,7 +94,22 @@ static inline void u64_stats_inc(u64_stats_t *p) ...@@ -98,7 +94,22 @@ static inline void u64_stats_inc(u64_stats_t *p)
local64_inc(&p->v); local64_inc(&p->v);
} }
#else static inline void u64_stats_init(struct u64_stats_sync *syncp) { }
static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { }
static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { }
static inline unsigned long __u64_stats_irqsave(void) { return 0; }
static inline void __u64_stats_irqrestore(unsigned long flags) { }
static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
{
return 0;
}
static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
unsigned int start)
{
return false;
}
#else /* 64 bit */
typedef struct { typedef struct {
u64 v; u64 v;
...@@ -123,123 +134,95 @@ static inline void u64_stats_inc(u64_stats_t *p) ...@@ -123,123 +134,95 @@ static inline void u64_stats_inc(u64_stats_t *p)
{ {
p->v++; p->v++;
} }
#endif
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))
#define u64_stats_init(syncp) seqcount_init(&(syncp)->seq)
#else
static inline void u64_stats_init(struct u64_stats_sync *syncp) static inline void u64_stats_init(struct u64_stats_sync *syncp)
{ {
seqcount_init(&syncp->seq);
} }
#endif
static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp)
{ {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) preempt_disable_nested();
if (IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_disable();
write_seqcount_begin(&syncp->seq); write_seqcount_begin(&syncp->seq);
#endif
} }
static inline void u64_stats_update_end(struct u64_stats_sync *syncp) static inline void __u64_stats_update_end(struct u64_stats_sync *syncp)
{ {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))
write_seqcount_end(&syncp->seq); write_seqcount_end(&syncp->seq);
if (IS_ENABLED(CONFIG_PREEMPT_RT)) preempt_enable_nested();
preempt_enable();
#endif
} }
static inline unsigned long static inline unsigned long __u64_stats_irqsave(void)
u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)
{ {
unsigned long flags = 0; unsigned long flags;
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) local_irq_save(flags);
if (IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_disable();
else
local_irq_save(flags);
write_seqcount_begin(&syncp->seq);
#endif
return flags; return flags;
} }
static inline void static inline void __u64_stats_irqrestore(unsigned long flags)
u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
unsigned long flags)
{ {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) local_irq_restore(flags);
write_seqcount_end(&syncp->seq);
if (IS_ENABLED(CONFIG_PREEMPT_RT))
preempt_enable();
else
local_irq_restore(flags);
#endif
} }
static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
{ {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))
return read_seqcount_begin(&syncp->seq); return read_seqcount_begin(&syncp->seq);
#else
return 0;
#endif
} }
static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp) static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
unsigned int start)
{ {
#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT)) return read_seqcount_retry(&syncp->seq, start);
preempt_disable();
#endif
return __u64_stats_fetch_begin(syncp);
} }
#endif /* !64 bit */
static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
unsigned int start)
{ {
#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) __u64_stats_update_begin(syncp);
return read_seqcount_retry(&syncp->seq, start); }
#else
return false; static inline void u64_stats_update_end(struct u64_stats_sync *syncp)
#endif {
__u64_stats_update_end(syncp);
}
static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)
{
unsigned long flags = __u64_stats_irqsave();
__u64_stats_update_begin(syncp);
return flags;
}
static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
unsigned long flags)
{
__u64_stats_update_end(syncp);
__u64_stats_irqrestore(flags);
}
static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
{
return __u64_stats_fetch_begin(syncp);
} }
static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
unsigned int start) unsigned int start)
{ {
#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT))
preempt_enable();
#endif
return __u64_stats_fetch_retry(syncp, start); return __u64_stats_fetch_retry(syncp, start);
} }
/* /* Obsolete interfaces */
* In case irq handlers can update u64 counters, readers can use following helpers
* - SMP 32bit arches use seqcount protection, irq safe.
* - UP 32bit must disable irqs.
* - 64bit have no problem atomically reading u64 values, irq safe.
*/
static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp) static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)
{ {
#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) return u64_stats_fetch_begin(syncp);
preempt_disable();
#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP)
local_irq_disable();
#endif
return __u64_stats_fetch_begin(syncp);
} }
static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp, static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,
unsigned int start) unsigned int start)
{ {
#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) return u64_stats_fetch_retry(syncp, start);
preempt_enable();
#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP)
local_irq_enable();
#endif
return __u64_stats_fetch_retry(syncp, start);
} }
#endif /* _LINUX_U64_STATS_SYNC_H */ #endif /* _LINUX_U64_STATS_SYNC_H */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7