Commit f59d68ee authored by Shaohua Wang's avatar Shaohua Wang

BUG#21102971 data corruption on arm64

The root cause is that x86 has a stronger memory model than the ARM
processors. And the GCC builtins didn't issue the correct fences when
setting/unsetting the lock word. In particular during the mutex release.

The solution is rewriting atomic TAS operations: replace '__sync_' by
'__atomic_' if possible.
Reviewed-by: default avatarSunny Bains      <sunny.bains@oracle.com>
Reviewed-by: default avatarBin Su           <bin.x.su@oracle.com>
Reviewed-by: default avatarDebarun Banerjee <debarun.banerjee@oracle.com>
Reviewed-by: default avatarKrunal Bauskar   <krunal.bauskar@oracle.com>
RB: 9782
RB: 9665
RB: 9783
parent 552b1c8a
# Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved. # Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
...@@ -85,12 +85,28 @@ IF(NOT CMAKE_CROSSCOMPILING) ...@@ -85,12 +85,28 @@ IF(NOT CMAKE_CROSSCOMPILING)
}" }"
HAVE_IB_GCC_ATOMIC_BUILTINS HAVE_IB_GCC_ATOMIC_BUILTINS
) )
CHECK_C_SOURCE_RUNS(
"#include<stdint.h>
int main()
{
unsigned char c;
__atomic_test_and_set(&c, __ATOMIC_ACQUIRE);
__atomic_clear(&c, __ATOMIC_RELEASE);
return(0);
}"
HAVE_IB_GCC_ATOMIC_TEST_AND_SET
)
ENDIF() ENDIF()
IF(HAVE_IB_GCC_ATOMIC_BUILTINS) IF(HAVE_IB_GCC_ATOMIC_BUILTINS)
ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_BUILTINS=1) ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_BUILTINS=1)
ENDIF() ENDIF()
IF(HAVE_IB_GCC_ATOMIC_TEST_AND_SET)
ADD_DEFINITIONS(-DHAVE_IB_GCC_ATOMIC_TEST_AND_SET=1)
ENDIF()
# either define HAVE_IB_ATOMIC_PTHREAD_T_GCC or not # either define HAVE_IB_ATOMIC_PTHREAD_T_GCC or not
IF(NOT CMAKE_CROSSCOMPILING) IF(NOT CMAKE_CROSSCOMPILING)
CHECK_C_SOURCE_RUNS( CHECK_C_SOURCE_RUNS(
......
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved. Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
...@@ -37,6 +37,20 @@ Created 9/6/1995 Heikki Tuuri ...@@ -37,6 +37,20 @@ Created 9/6/1995 Heikki Tuuri
#include "univ.i" #include "univ.i"
#include "ut0lst.h" #include "ut0lst.h"
#if defined __i386__ || defined __x86_64__ || defined _M_IX86 \
|| defined _M_X64 || defined __WIN__
#define IB_STRONG_MEMORY_MODEL
#endif /* __i386__ || __x86_64__ || _M_IX86 || M_X64 || __WIN__ */
#ifdef HAVE_WINDOWS_ATOMICS
typedef LONG lock_word_t; /*!< On Windows, InterlockedExchange operates
on LONG variable */
#else
typedef byte lock_word_t;
#endif
#ifdef __WIN__ #ifdef __WIN__
/** Native event (slow)*/ /** Native event (slow)*/
typedef HANDLE os_native_event_t; typedef HANDLE os_native_event_t;
...@@ -304,11 +318,61 @@ amount of increment. */ ...@@ -304,11 +318,61 @@ amount of increment. */
# define os_atomic_increment_ulint(ptr, amount) \ # define os_atomic_increment_ulint(ptr, amount) \
os_atomic_increment(ptr, amount) os_atomic_increment(ptr, amount)
/**********************************************************//** # if defined(HAVE_IB_GCC_ATOMIC_TEST_AND_SET)
Returns the old value of *ptr, atomically sets *ptr to new_val */
/** Do an atomic test-and-set.
@param[in,out] ptr Memory location to set to non-zero
@return the previous value */
static inline
lock_word_t
os_atomic_test_and_set(volatile lock_word_t* ptr)
{
return(__atomic_test_and_set(ptr, __ATOMIC_ACQUIRE));
}
/** Do an atomic clear.
@param[in,out] ptr Memory location to set to zero */
static inline
void
os_atomic_clear(volatile lock_word_t* ptr)
{
__atomic_clear(ptr, __ATOMIC_RELEASE);
}
# elif defined(IB_STRONG_MEMORY_MODEL)
/** Do an atomic test and set.
@param[in,out] ptr Memory location to set to non-zero
@return the previous value */
static inline
lock_word_t
os_atomic_test_and_set(volatile lock_word_t* ptr)
{
return(__sync_lock_test_and_set(ptr, 1));
}
/** Do an atomic release.
In theory __sync_lock_release should be used to release the lock.
Unfortunately, it does not work properly alone. The workaround is
that more conservative __sync_lock_test_and_set is used instead.
Performance regression was observed at some conditions for Intel
architecture. Disable release barrier on Intel architecture for now.
@param[in,out] ptr Memory location to write to
@return the previous value */
static inline
lock_word_t
os_atomic_clear(volatile lock_word_t* ptr)
{
return(__sync_lock_test_and_set(ptr, 0));
}
# define os_atomic_test_and_set_byte(ptr, new_val) \ # else
__sync_lock_test_and_set(ptr, (byte) new_val)
# error "Unsupported platform"
# endif /* HAVE_IB_GCC_ATOMIC_TEST_AND_SET */
#elif defined(HAVE_IB_SOLARIS_ATOMICS) #elif defined(HAVE_IB_SOLARIS_ATOMICS)
...@@ -357,11 +421,25 @@ amount of increment. */ ...@@ -357,11 +421,25 @@ amount of increment. */
# define os_atomic_increment_ulint(ptr, amount) \ # define os_atomic_increment_ulint(ptr, amount) \
atomic_add_long_nv(ptr, amount) atomic_add_long_nv(ptr, amount)
/**********************************************************//** /** Do an atomic xchg and set to non-zero.
Returns the old value of *ptr, atomically sets *ptr to new_val */ @param[in,out] ptr Memory location to set to non-zero
@return the previous value */
# define os_atomic_test_and_set_byte(ptr, new_val) \ static inline
atomic_swap_uchar(ptr, new_val) lock_word_t
os_atomic_test_and_set(volatile lock_word_t* ptr)
{
return(atomic_swap_uchar(ptr, 1));
}
/** Do an atomic xchg and set to zero.
@param[in,out] ptr Memory location to set to zero
@return the previous value */
static inline
lock_word_t
os_atomic_clear(volatile lock_word_t* ptr)
{
return(atomic_swap_uchar(ptr, 0));
}
#elif defined(HAVE_WINDOWS_ATOMICS) #elif defined(HAVE_WINDOWS_ATOMICS)
...@@ -403,13 +481,27 @@ amount of increment. */ ...@@ -403,13 +481,27 @@ amount of increment. */
# define os_atomic_increment_ulint(ptr, amount) \ # define os_atomic_increment_ulint(ptr, amount) \
((ulint) (win_xchg_and_add(ptr, amount) + amount)) ((ulint) (win_xchg_and_add(ptr, amount) + amount))
/**********************************************************//** /** Do an atomic test and set.
Returns the old value of *ptr, atomically sets *ptr to new_val. InterlockedExchange() operates on LONG, and the LONG will be clobbered
InterlockedExchange() operates on LONG, and the LONG will be @param[in,out] ptr Memory location to set to non-zero
clobbered */ @return the previous value */
static inline
# define os_atomic_test_and_set_byte(ptr, new_val) \ lock_word_t
((byte) InterlockedExchange(ptr, new_val)) os_atomic_test_and_set(volatile lock_word_t* ptr)
{
return(InterlockedExchange(ptr, 1));
}
/** Do an atomic release.
InterlockedExchange() operates on LONG, and the LONG will be clobbered
@param[in,out] ptr Memory location to set to zero
@return the previous value */
static inline
lock_word_t
os_atomic_clear(volatile lock_word_t* ptr)
{
return(InterlockedExchange(ptr, 0));
}
#else #else
# define IB_ATOMICS_STARTUP_MSG \ # define IB_ATOMICS_STARTUP_MSG \
......
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
...@@ -45,13 +45,6 @@ Created 9/5/1995 Heikki Tuuri ...@@ -45,13 +45,6 @@ Created 9/5/1995 Heikki Tuuri
extern my_bool timed_mutexes; extern my_bool timed_mutexes;
#endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */ #endif /* UNIV_DEBUG && !UNIV_HOTBACKUP */
#ifdef HAVE_WINDOWS_ATOMICS
typedef LONG lock_word_t; /*!< On Windows, InterlockedExchange operates
on LONG variable */
#else
typedef byte lock_word_t;
#endif
#if defined UNIV_PFS_MUTEX || defined UNIV_PFS_RWLOCK #if defined UNIV_PFS_MUTEX || defined UNIV_PFS_RWLOCK
/* There are mutexes/rwlocks that we want to exclude from /* There are mutexes/rwlocks that we want to exclude from
instrumentation even if their corresponding performance schema instrumentation even if their corresponding performance schema
......
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved. Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
...@@ -80,7 +80,7 @@ mutex_test_and_set( ...@@ -80,7 +80,7 @@ mutex_test_and_set(
mutex_t* mutex) /*!< in: mutex */ mutex_t* mutex) /*!< in: mutex */
{ {
#if defined(HAVE_ATOMIC_BUILTINS) #if defined(HAVE_ATOMIC_BUILTINS)
return(os_atomic_test_and_set_byte(&mutex->lock_word, 1)); return(os_atomic_test_and_set(&mutex->lock_word));
#else #else
ibool ret; ibool ret;
...@@ -108,10 +108,7 @@ mutex_reset_lock_word( ...@@ -108,10 +108,7 @@ mutex_reset_lock_word(
mutex_t* mutex) /*!< in: mutex */ mutex_t* mutex) /*!< in: mutex */
{ {
#if defined(HAVE_ATOMIC_BUILTINS) #if defined(HAVE_ATOMIC_BUILTINS)
/* In theory __sync_lock_release should be used to release the lock. os_atomic_clear(&mutex->lock_word);
Unfortunately, it does not work properly alone. The workaround is
that more conservative __sync_lock_test_and_set is used instead. */
os_atomic_test_and_set_byte(&mutex->lock_word, 0);
#else #else
mutex->lock_word = 0; mutex->lock_word = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment