coverted my_thread_init() /end to use my_malloc()/my_free() to help track

down replication corruption

dbug/dbug.c

@@ -21,7 +21,8 @@
  *	all copies and derivative works.  Thank you.			      *
  *									      *
  *	The author makes no warranty of any kind  with	respect  to  this     *
- *	product  and  explicitly disclaims any implied warranties of mer-     *
+ *	product  and  explicitly disclaims any implied warranties of mer-     *ct_lex.table_list.first=0;
+  thd->lex.selec
  *	chantability or fitness for any particular purpose.		      *
  *									      *
  ******************************************************************************
@@ -58,7 +59,7 @@
  *	seismo!bpa!sjuvax!bbanerje
  *
  *	Michael Widenius:
- *	DBUG_DUMP	- To dump a pice of memory.
+ *	DBUG_DUMP	- To dump a block of memory.
  *	PUSH_FLAG "O"	- To be used insted of "o" if we don't
  *			  want flushing (for slow systems)
  *	PUSH_FLAG "A"	- as 'O', but we will append to the out file instead
@@ -707,7 +708,13 @@ char ***_sframep_ __attribute__((unused)))
     int save_errno=errno;
     if (!init_done)
       _db_push_ (_DBUG_START_CONDITION_);
-    state=code_state();
+    /* Sasha: the test below is so we could call functions with DBUG_ENTER
+       before my_thread_init(). I needed this because I suspected corruption
+       of a block allocated by my_thread_init() itself, so I wanted to use
+       my_malloc()/my_free() in my_thread_init()/my_thread_end()
+    */
+    if (!(state=code_state()))
+      return;
 
     *_sfunc_ = state->func;
     *_sfile_ = state->file;
@@ -855,6 +862,9 @@ uint _line_,
 const char *keyword)
 {
   CODE_STATE *state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!state)
+    return;
   state->u_line = _line_;
   state->u_keyword = (char*) keyword;
 }
@@ -890,7 +900,9 @@ void _db_doprnt_ (const char *format,...)
 {
   va_list args;
   CODE_STATE *state;
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return;
 
   va_start(args,format);
 
@@ -942,7 +954,9 @@ uint length)
   int pos;
   char dbuff[90];
   CODE_STATE *state;
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return;
 
   if (_db_keyword_ ((char*) keyword))
   {
@@ -1224,7 +1238,9 @@ const char *keyword)
 
   if (!init_done)
     _db_push_ ("");
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return FALSE;
   result = FALSE;
   if (DEBUGGING &&
       state->level <= stack -> maxdepth &&

include/my_sys.h

@@ -137,6 +137,10 @@ extern int NEAR my_errno;		/* Last error in mysys */
 #define NORMAL_SAFEMALLOC sf_malloc_quick=0
 extern uint sf_malloc_prehunc,sf_malloc_endhunc,sf_malloc_quick;
 extern ulonglong safemalloc_mem_limit;
+/* keep track of shutdown,signal, and main threads so that my_end() will not
+   report errors with them
+*/
+extern pthread_t shutdown_th, main_th,signal_th;
 #define CALLER_INFO_PROTO   , const char *sFile, uint uLine
 #define CALLER_INFO         , __FILE__, __LINE__
 #define ORIG_CALLER_INFO    , sFile, uLine

mysys/my_static.h

@@ -38,6 +38,7 @@ struct irem {
     my_string _sFileName;		/* File in which memory was new'ed */
     uint _uLineNum;			/* Line number in above file	   */
     uint _uDataSize;			/* Size requested		   */
+    pthread_t thread_id;
     long _lSpecialValue;		/* Underrun marker value	   */
 };
 
@@ -56,6 +57,11 @@ extern const char *soundex_map;
 extern USED_MEM* my_once_root_block;
 extern uint	 my_once_extra;
 
+/* these threads are exept from safemalloc leak scrutiny unless
+   PEDANTIC_SAFEMALLOC is defined
+*/
+extern pthread_t signal_thread,kill_thread;
+
 #ifndef HAVE_TEMPNAM
 extern int	_my_tempnam_used;
 #endif

mysys/my_thr_init.c

@@ -105,19 +105,33 @@ static long thread_id=0;
 my_bool my_thread_init(void)
 {
   struct st_my_thread_var *tmp;
+#ifdef EXTRA_DEBUG
+  fprintf(stderr,"my_thread_init(): thread_id=%ld\n",pthread_self());
+#endif  
 #if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX)
   pthread_mutex_lock(&THR_LOCK_lock);
 #endif
 #if !defined(__WIN__) || defined(USE_TLS)
   if (my_pthread_getspecific(struct st_my_thread_var *,THR_KEY_mysys))
   {
+#ifdef EXTRA_DEBUG
+    fprintf(stderr,"my_thread_init() called more than once in thread %ld\n",
+	    pthread_self());
+#endif    
     pthread_mutex_unlock(&THR_LOCK_lock);
     return 0;						/* Safequard */
   }
     /* We must have many calloc() here because these are freed on
        pthread_exit */
+    /*
+      Sasha: the above comment does not make sense. I have changed calloc() to
+      equivalent my_malloc() but it was calloc() before. It seems like the
+      comment is out of date - we always call my_thread_end() before
+      pthread_exit() to clean up. Note that I have also fixed up DBUG
+      code to be able to call it from my_thread_init()
+     */
   if (!(tmp=(struct st_my_thread_var *)
-	calloc(1,sizeof(struct st_my_thread_var))))
+	my_malloc(sizeof(struct st_my_thread_var),MYF(MY_WME|MY_ZEROFILL))))
   {
     pthread_mutex_unlock(&THR_LOCK_lock);
     return 1;
@@ -125,6 +139,9 @@ my_bool my_thread_init(void)
   pthread_setspecific(THR_KEY_mysys,tmp);
 
 #else
+  /* Sasha: TODO - explain what exactly we are doing on Windows
+     At first glance, I have a hard time following the code
+   */
   if (THR_KEY_mysys.id)   /* Already initialized */
   {
 #if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX)
@@ -146,9 +163,18 @@ my_bool my_thread_init(void)
 void my_thread_end(void)
 {
   struct st_my_thread_var *tmp=my_thread_var;
+#ifdef EXTRA_DEBUG
+  fprintf(stderr,"my_thread_end(): tmp=%p,thread_id=%ld\n",
+	  tmp,pthread_self());
+#endif  
   if (tmp)
   {
 #if !defined(DBUG_OFF)
+    /* Sasha:  tmp->dbug is allocated inside DBUG library
+       so for now we will not mess with trying to use my_malloc()/
+       my_free(), but in the future it would be nice to figure out a
+       way to do it
+    */
     if (tmp->dbug)
     {
       free(tmp->dbug);
@@ -160,12 +186,15 @@ void my_thread_end(void)
 #endif
     pthread_mutex_destroy(&tmp->mutex);
 #if (!defined(__WIN__) && !defined(OS2)) || defined(USE_TLS)
-    free(tmp);
+    /* we need to setspecific to 0 BEFORE we call my_free, as my_free
+       uses some DBUG_ macros that will use the follow the specific
+       pointer after the block it is pointing to has been freed if
+       specific does not get reset first
+    */
+    pthread_setspecific(THR_KEY_mysys,0);
+    my_free((gptr)tmp,MYF(MY_WME));
 #endif
   }
-#if (!defined(__WIN__) && !defined(OS2)) || defined(USE_TLS)
-  pthread_setspecific(THR_KEY_mysys,0);
-#endif
 }
 
 struct st_my_thread_var *_my_thread_var(void)

mysys/safemalloc.c

@@ -73,14 +73,25 @@
 #include "mysys_err.h"
 
 ulonglong safemalloc_mem_limit = ~(ulonglong)0;
+pthread_t shutdown_th=0,main_th=0,signal_th=0;
 
 #define pNext		tInt._pNext
 #define pPrev		tInt._pPrev
 #define sFileName	tInt._sFileName
 #define uLineNum	tInt._uLineNum
 #define uDataSize	tInt._uDataSize
+#define thread_id       tInt.thread_id
 #define lSpecialValue	tInt._lSpecialValue
 
+#ifndef PEDANTIC_SAFEMALLOC
+static int sf_malloc_tampered = 0; /* set to 1 after TERMINATE() if we had
+				    to fiddle with cNewCount and the linked
+				    list of blocks so that _sanity() will
+				    not fuss when it is not supposed to
+				 */
+#endif
+				   
+
 	/* Static functions prototypes */
 
 static int check_ptr(const char *where, byte *ptr, const char *sFile,
@@ -174,6 +185,7 @@ gptr _mymalloc (uint uSize, const char *sFile, uint uLine, myf MyFlags)
     pTmp -> sFileName = (my_string) sFile;
     pTmp -> uLineNum = uLine;
     pTmp -> uDataSize = uSize;
+    pTmp->thread_id = pthread_self();
     pTmp -> pPrev = NULL;
 
     /* Add this remember structure to the linked list */
@@ -359,6 +371,12 @@ static int check_ptr(const char *where, byte *ptr, const char *sFile,
   return 0;
 }
 
+static int legal_leak(struct remember* pPtr)
+{
+ return pthread_self() == pPtr->thread_id || main_th == pPtr->thread_id
+	    || shutdown_th == pPtr->thread_id
+   || signal_th == pPtr->thread_id;
+}
 
 /*
  * TERMINATE(FILE *file)
@@ -376,6 +394,47 @@ void TERMINATE (FILE *file)
   /* NEW and the number of calls to FREE.  >0 means more	 */
   /* NEWs than FREEs.  <0, etc.				 */
 
+#ifndef PEDANTIC_SAFEMALLOC
+  /* Avoid false alarms for blocks that we cannot free before my_end()
+     This does miss some positives, but that is ok. This will only miss
+     failures to free things allocated in the main thread which 
+     performs only one-time allocations. If you really need to
+     debug memory allocations in the main thread,
+     #define PEDANTIC_SAFEMALLOC
+  */
+  if ((pPtr=pRememberRoot))
+  {
+    while (pPtr)
+    {
+      if (legal_leak(pPtr))
+      {
+	sf_malloc_tampered=1;
+	cNewCount--;
+	lCurMemory -= pPtr->uDataSize;
+	if (pPtr->pPrev)
+	{
+	  struct remember* tmp;
+	  tmp = pPtr->pPrev->pNext = pPtr->pNext;
+	  if (tmp)
+	   tmp->pPrev = pPtr->pPrev; 
+	  pPtr->pNext = pPtr->pPrev = 0;
+	  pPtr = tmp;
+	}
+	else
+	{
+	  pRememberRoot = pPtr->pNext;
+	  pPtr->pNext = pPtr->pPrev = 0;
+	  pPtr = pRememberRoot;
+	  if (pPtr)
+	    pPtr->pPrev=0;
+	}
+      }
+      else
+	pPtr = pPtr->pNext;
+    }
+  }
+#endif  
+  
   if (cNewCount)
   {
     if (file)
@@ -402,10 +461,14 @@ void TERMINATE (FILE *file)
       if (file)
       {
 	fprintf (file,
-		 "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'\n",
+		 "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'",
 		 pPtr -> uDataSize,
 		 (ulong) &(pPtr -> aData[sf_malloc_prehunc]),
 		 pPtr -> uLineNum, pPtr -> sFileName);
+#ifdef THREAD
+	fprintf(file, " in thread %ld", pPtr->thread_id);
+#endif
+	fprintf(file, "\n");
 	(void) fflush(file);
       }
       DBUG_PRINT("safe",
@@ -484,6 +547,10 @@ int _sanity (const char *sFile, uint uLine)
   uint count=0;
 
   pthread_mutex_lock(&THR_LOCK_malloc);
+#ifndef PEDANTIC_SAFEMALLOC  
+  if (sf_malloc_tampered && cNewCount < 0)
+    cNewCount=0;
+#endif  
   count=cNewCount;
   for (pTmp = pRememberRoot; pTmp != NULL && count-- ; pTmp = pTmp -> pNext)
     flag+=_checkchunk (pTmp, sFile, uLine);
@@ -492,6 +559,7 @@ int _sanity (const char *sFile, uint uLine)
   {
     const char *format="Safemalloc link list destroyed, discovered at '%s:%d'";
     fprintf (stderr, format, sFile, uLine); fputc('\n',stderr);
+    fprintf (stderr, "root=%p,count=%d,pTmp=%p\n", pRememberRoot,count,pTmp);
     (void) fflush(stderr);
     DBUG_PRINT("safe",(format, sFile, uLine));
     flag=1;

sql/mysqld.cc

@@ -38,7 +38,17 @@
 #define ONE_THREAD
 #endif
 
-/* do stack traces are only supported on linux intel */
+#ifdef SAFEMALLOC
+#define SHUTDOWN_THD shutdown_th=pthread_self();
+#define MAIN_THD main_th=pthread_self();
+#define SIGNAL_THD signal_th=pthread_self();
+#else
+#define SHUTDOWN_THD
+#define MAIN_THD
+#define SIGNAL_THD
+#endif
+
+/* stack traces are only supported on linux intel */
 #if defined(__linux__)  && defined(__i386__) && defined(USE_PSTACK)
 #define	HAVE_STACK_TRACE_ON_SEGV
 #include "../pstack/pstack.h"
@@ -694,6 +704,7 @@ static void __cdecl kill_server(int sig_ptr)
     sql_print_error(ER(ER_GOT_SIGNAL),my_progname,sig); /* purecov: inspected */
 
 #if defined(USE_ONE_SIGNAL_HAND) && !defined(__WIN__) && !defined(OS2)
+  SHUTDOWN_THD;
   my_thread_init();				// If this is a new thread
 #endif
   close_connections();
@@ -709,6 +720,7 @@ static void __cdecl kill_server(int sig_ptr)
 #ifdef USE_ONE_SIGNAL_HAND
 static pthread_handler_decl(kill_server_thread,arg __attribute__((unused)))
 {
+  SHUTDOWN_THD;
   my_thread_init();				// Initialize new thread
   kill_server(0);
   my_thread_end();				// Normally never reached
@@ -1252,6 +1264,7 @@ static void init_signals(void)
   signal(SIGALRM, SIG_IGN);
   signal(SIGBREAK,SIG_IGN);
   signal_thread = pthread_self();
+  SIGNAL_THD;
 }
 
 static void start_signal_handler(void)
@@ -1445,7 +1458,7 @@ static void *signal_hand(void *arg __attribute__((unused)))
   int sig;
   my_thread_init();				// Init new thread
   DBUG_ENTER("signal_hand");
-
+  SIGNAL_THD;
   /* Setup alarm handler */
   init_thr_alarm(max_connections+max_insert_delayed_threads);
 #if SIGINT != THR_KILL_SIGNAL
@@ -1500,7 +1513,10 @@ static void *signal_hand(void *arg __attribute__((unused)))
     else
       while ((error=my_sigwait(&set,&sig)) == EINTR) ;
     if (cleanup_done)
+    {
+      my_thread_end();
       pthread_exit(0);				// Safety
+    }
     switch (sig) {
     case SIGTERM:
     case SIGQUIT:
@@ -1594,6 +1610,7 @@ int uname(struct utsname *a)
 pthread_handler_decl(handle_shutdown,arg)
 {
   MSG msg;
+  SHUTDOWN_THD;
   my_thread_init();
 
   /* this call should create the message queue for this thread */
@@ -1620,6 +1637,7 @@ int __stdcall handle_kill(ulong ctrl_type)
 #ifdef OS2
 pthread_handler_decl(handle_shutdown,arg)
 {
+  SHUTDOWN_THD;
   my_thread_init();
 
   // wait semaphore
@@ -1691,6 +1709,7 @@ int main(int argc, char **argv)
 
   my_umask=0660;		// Default umask for new files
   my_umask_dir=0700;		// Default umask for new directories
+  MAIN_THD;
   MY_INIT(argv[0]);		// init my_sys library & pthreads
   tzset();			// Set tzname