Commit 8c8b096a authored by Peter Zijlstra's avatar Peter Zijlstra

instrumentation: Wire up cmpxchg128()

Wire up the cmpxchg128 family in the atomic wrapper scripts.

These provide the generic cmpxchg128 family of functions from the
arch_ prefixed version, adding explicit instrumentation where needed.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarArnd Bergmann <arnd@arndb.de>
Reviewed-by: default avatarMark Rutland <mark.rutland@arm.com>
Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Tested-by: default avatarMark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230531132323.519237070@infradead.org
parent b23e139d
...@@ -77,6 +77,29 @@ ...@@ -77,6 +77,29 @@
#endif /* arch_cmpxchg64_relaxed */ #endif /* arch_cmpxchg64_relaxed */
#ifndef arch_cmpxchg128_relaxed
#define arch_cmpxchg128_acquire arch_cmpxchg128
#define arch_cmpxchg128_release arch_cmpxchg128
#define arch_cmpxchg128_relaxed arch_cmpxchg128
#else /* arch_cmpxchg128_relaxed */
#ifndef arch_cmpxchg128_acquire
#define arch_cmpxchg128_acquire(...) \
__atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
#endif
#ifndef arch_cmpxchg128_release
#define arch_cmpxchg128_release(...) \
__atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
#endif
#ifndef arch_cmpxchg128
#define arch_cmpxchg128(...) \
__atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
#endif
#endif /* arch_cmpxchg128_relaxed */
#ifndef arch_try_cmpxchg_relaxed #ifndef arch_try_cmpxchg_relaxed
#ifdef arch_try_cmpxchg #ifdef arch_try_cmpxchg
#define arch_try_cmpxchg_acquire arch_try_cmpxchg #define arch_try_cmpxchg_acquire arch_try_cmpxchg
...@@ -217,6 +240,76 @@ ...@@ -217,6 +240,76 @@
#endif /* arch_try_cmpxchg64_relaxed */ #endif /* arch_try_cmpxchg64_relaxed */
#ifndef arch_try_cmpxchg128_relaxed
#ifdef arch_try_cmpxchg128
#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128
#define arch_try_cmpxchg128_release arch_try_cmpxchg128
#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128
#endif /* arch_try_cmpxchg128 */
#ifndef arch_try_cmpxchg128
#define arch_try_cmpxchg128(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128 */
#ifndef arch_try_cmpxchg128_acquire
#define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_acquire */
#ifndef arch_try_cmpxchg128_release
#define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_release */
#ifndef arch_try_cmpxchg128_relaxed
#define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_relaxed */
#else /* arch_try_cmpxchg128_relaxed */
#ifndef arch_try_cmpxchg128_acquire
#define arch_try_cmpxchg128_acquire(...) \
__atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg128_release
#define arch_try_cmpxchg128_release(...) \
__atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg128
#define arch_try_cmpxchg128(...) \
__atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
#endif
#endif /* arch_try_cmpxchg128_relaxed */
#ifndef arch_try_cmpxchg_local #ifndef arch_try_cmpxchg_local
#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ #define arch_try_cmpxchg_local(_ptr, _oldp, _new) \
({ \ ({ \
...@@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) ...@@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
#endif #endif
#endif /* _LINUX_ATOMIC_FALLBACK_H */ #endif /* _LINUX_ATOMIC_FALLBACK_H */
// ad2e2b4d168dbc60a73922616047a9bfa446af36 // 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53
...@@ -2034,6 +2034,36 @@ atomic_long_dec_if_positive(atomic_long_t *v) ...@@ -2034,6 +2034,36 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
}) })
#define cmpxchg128(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
})
#define cmpxchg128_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
})
#define try_cmpxchg(ptr, oldp, ...) \ #define try_cmpxchg(ptr, oldp, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
...@@ -2110,6 +2140,44 @@ atomic_long_dec_if_positive(atomic_long_t *v) ...@@ -2110,6 +2140,44 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
}) })
#define try_cmpxchg128(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define cmpxchg_local(ptr, ...) \ #define cmpxchg_local(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
...@@ -2124,6 +2192,13 @@ atomic_long_dec_if_positive(atomic_long_t *v) ...@@ -2124,6 +2192,13 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
}) })
#define cmpxchg128_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
})
#define sync_cmpxchg(ptr, ...) \ #define sync_cmpxchg(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
...@@ -2150,6 +2225,15 @@ atomic_long_dec_if_positive(atomic_long_t *v) ...@@ -2150,6 +2225,15 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
}) })
#define try_cmpxchg128_local(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define cmpxchg_double(ptr, ...) \ #define cmpxchg_double(ptr, ...) \
({ \ ({ \
typeof(ptr) __ai_ptr = (ptr); \ typeof(ptr) __ai_ptr = (ptr); \
...@@ -2167,4 +2251,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) ...@@ -2167,4 +2251,4 @@ atomic_long_dec_if_positive(atomic_long_t *v)
}) })
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e // 82d1be694fab30414527d0877c29fa75ed5a0b74
...@@ -217,11 +217,11 @@ cat << EOF ...@@ -217,11 +217,11 @@ cat << EOF
EOF EOF
for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do
gen_xchg_fallbacks "${xchg}" gen_xchg_fallbacks "${xchg}"
done done
for cmpxchg in "cmpxchg" "cmpxchg64"; do for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
gen_try_cmpxchg_fallbacks "${cmpxchg}" gen_try_cmpxchg_fallbacks "${cmpxchg}"
done done
......
...@@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name meta args; do ...@@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name meta args; do
done done
for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
for order in "" "_acquire" "_release" "_relaxed"; do for order in "" "_acquire" "_release" "_relaxed"; do
gen_xchg "${xchg}" "${order}" "" gen_xchg "${xchg}" "${order}" ""
printf "\n" printf "\n"
done done
done done
for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
gen_xchg "${xchg}" "" "" gen_xchg "${xchg}" "" ""
printf "\n" printf "\n"
done done
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment