ndb

  new testprogram testSRBank


BitKeeper/etc/ignore:
  Added ndb/test/ndbapi/testSRBank to the ignore list
ndb/test/include/NDBT_Test.hpp:
  incProperty
ndb/test/ndbapi/Makefile.am:
  new test program
ndb/test/ndbapi/bank/Bank.cpp:
  remove endless wait until ready loop
  fix lots of retry loops to better handle error
ndb/test/ndbapi/bank/Bank.hpp:
  remove endless wait until ready loop
  fix lots of retry loops to better handle error
ndb/test/src/NDBT_Test.cpp:
  incProperty
ndb/test/src/UtilTransactions.cpp:
  close transaction if failed
ndb/test/ndbapi/testSRBank.cpp:
  New BitKeeper file ``ndb/test/ndbapi/testSRBank.cpp''

.bzrignore

@@ -1056,3 +1056,4 @@ vio/viotest-ssl
 ndb/tools/ndb_config
 support-files/MacOSX/postflight
 support-files/MacOSX/preflight
+ndb/test/ndbapi/testSRBank

ndb/test/include/NDBT_Test.hpp

@@ -64,6 +64,7 @@ public:
   const char* getPropertyWait(const char*, const char* );
 
   void decProperty(const char *);
+  void incProperty(const char *);
 
   // Communicate with other tests
   void stopTest();

ndb/test/ndbapi/Makefile.am

@@ -31,7 +31,8 @@ testTimeout \
 testTransactions \
 testDeadlock \
 test_event ndbapi_slow_select testReadPerf testLcp \
-DbCreate DbAsyncGenerator
+DbCreate DbAsyncGenerator \
+testSRBank
 
 #flexTimedAsynch
 #testBlobs
@@ -72,6 +73,7 @@ testReadPerf_SOURCES = testReadPerf.cpp
 testLcp_SOURCES = testLcp.cpp
 DbCreate_SOURCES= bench/mainPopulate.cpp bench/dbPopulate.cpp bench/userInterface.cpp bench/dbPopulate.h bench/userInterface.h bench/testData.h bench/testDefinitions.h bench/ndb_schema.hpp bench/ndb_error.hpp
 DbAsyncGenerator_SOURCES= bench/mainAsyncGenerator.cpp bench/asyncGenerator.cpp bench/ndb_async2.cpp bench/dbGenerator.h bench/macros.h bench/userInterface.h bench/testData.h bench/testDefinitions.h bench/ndb_schema.hpp bench/ndb_error.hpp
+testSRBank_SOURCES = testSRBank.cpp
 
 INCLUDES_LOC = -I$(top_srcdir)/ndb/include/kernel
 
@@ -83,6 +85,7 @@ include $(top_srcdir)/ndb/config/type_ndbapitest.mk.am
 ##testSystemRestart_INCLUDES = $(INCLUDES) -I$(top_srcdir)/ndb/include/kernel
 ##testTransactions_INCLUDES = $(INCLUDES) -I$(top_srcdir)/ndb/include/kernel
 testBackup_LDADD = $(LDADD) bank/libbank.a
+testSRBank_LDADD = bank/libbank.a $(LDADD) 
 
 # Don't update the files from bitkeeper
 %::SCCS/s.%

ndb/test/ndbapi/bank/Bank.cpp

@@ -19,12 +19,13 @@
 #include <NdbSleep.h>
 #include <UtilTransactions.hpp>
 
-Bank::Bank():
+Bank::Bank(bool _init):
   m_ndb("BANK"),
   m_maxAccount(-1),
   m_initialized(false)
 {
-
+  if(_init)
+    init();
 }
 
 int Bank::init(){
@@ -34,26 +35,25 @@ int Bank::init(){
   myRandom48Init(NdbTick_CurrentMillisecond());
 
   m_ndb.init();   
-  while (m_ndb.waitUntilReady(10) != 0)
-    ndbout << "Waiting for ndb to be ready" << endl;
+  if (m_ndb.waitUntilReady(30) != 0)
+  {
+    ndbout << "Ndb not ready" << endl;
+    return NDBT_FAILED;
+  }
   
   if (getNumAccounts() != NDBT_OK)
     return NDBT_FAILED;
+
+  m_initialized = true;
   return NDBT_OK;
 }
 
 int Bank::performTransactions(int maxSleepBetweenTrans, int yield){
 
-  if (init() != NDBT_OK)
-    return NDBT_FAILED;
   int transactions = 0;
 
-  while(1){
-
-    while(m_ndb.waitUntilReady(10) != 0)
-      ndbout << "Waiting for ndb to be ready" << endl;
-
-    while(performTransaction() != NDBT_FAILED){
+  while(performTransaction() == NDBT_OK)
+  {
     transactions++;
     
     if (maxSleepBetweenTrans > 0){
@@ -67,7 +67,7 @@ int Bank::performTransactions(int maxSleepBetweenTrans, int yield){
     if (yield != 0 && transactions >= yield)
       return NDBT_OK;
   }
-  }
+
   return NDBT_FAILED;
 
 }
@@ -92,7 +92,7 @@ int Bank::performTransaction(){
     
   int amount = myRandom48(maxAmount);
 
- retry_transaction:
+retry_transaction:
   int res = performTransaction(fromAccount, toAccount, amount);
   if (res != 0){
     switch (res){
@@ -158,8 +158,9 @@ int Bank::performTransactionImpl1(int fromAccountId,
     
   // Ok, all clear to do the transaction
   Uint64 transId;
-  if (getNextTransactionId(transId) != NDBT_OK){
-    return NDBT_FAILED;
+  int result = NDBT_OK;
+  if ((result= getNextTransactionId(transId)) != NDBT_OK){
+    return result;
   }
 
   NdbConnection* pTrans = m_ndb.startTransaction();
@@ -500,8 +501,6 @@ int Bank::performTransactionImpl1(int fromAccountId,
 
 int Bank::performMakeGLs(int yield){
   int result;
-  if (init() != NDBT_OK)
-    return NDBT_FAILED;
   
   int counter, maxCounter;
   int yieldCounter = 0;
@@ -512,9 +511,6 @@ int Bank::performMakeGLs(int yield){
     counter = 0;
     maxCounter = 50 + myRandom48(100);
     
-    while(m_ndb.waitUntilReady(10) != 0)
-      ndbout << "Waiting for ndb to be ready" << endl;
-
     /** 
      * Validate GLs and Transactions for previous days
      *
@@ -526,6 +522,7 @@ int Bank::performMakeGLs(int yield){
 	return NDBT_FAILED;
       }
       g_info << "performValidateGLs failed" << endl;
+      return NDBT_FAILED;
       continue;
     }
 
@@ -536,7 +533,7 @@ int Bank::performMakeGLs(int yield){
 	return NDBT_FAILED;
       }
       g_info << "performValidatePurged failed" << endl;
-      continue;
+      return NDBT_FAILED;
     }
 
     while (1){
@@ -607,14 +604,9 @@ int Bank::performMakeGLs(int yield){
 
 int Bank::performValidateAllGLs(){
   int result;
-  if (init() != NDBT_OK)
-    return NDBT_FAILED;
   
   while (1){
     
-    while(m_ndb.waitUntilReady(10) != 0)
-      ndbout << "Waiting for ndb to be ready" << endl;
-
     /** 
      * Validate GLs and Transactions for previous days
      * Set age so that ALL GL's are validated
@@ -1937,17 +1929,10 @@ int Bank::findTransactionsToPurge(const Uint64 glTime,
 }
  
  
- int Bank::performIncreaseTime(int maxSleepBetweenDays, int yield){
-  if (init() != NDBT_OK)
-    return NDBT_FAILED;
-
+int Bank::performIncreaseTime(int maxSleepBetweenDays, int yield)
+{
   int yieldCounter = 0;
   
-   while(1){
-     
-     while(m_ndb.waitUntilReady(10) != 0)
-      ndbout << "Waiting for ndb to be ready" << endl;
-     
   while(1){
     
     Uint64 currTime;
@@ -1965,11 +1950,8 @@ int Bank::findTransactionsToPurge(const Uint64 glTime,
       return NDBT_OK;
     
   }
-   }
   return NDBT_FAILED;
- }
-
-
+}
 
 int Bank::readSystemValue(SystemValueId sysValId, Uint64 & value){
 
@@ -1978,18 +1960,26 @@ int Bank::readSystemValue(SystemValueId sysValId, Uint64 & value){
   NdbConnection* pTrans = m_ndb.startTransaction();
   if (pTrans == NULL){
     ERR(m_ndb.getNdbError());
+    if(m_ndb.getNdbError().status == NdbError::TemporaryError)
+      return NDBT_TEMPORARY;
     return NDBT_FAILED;
   }
 
-  if (prepareReadSystemValueOp(pTrans, sysValId, value) != NDBT_OK) {
+  int result;
+  if ((result= prepareReadSystemValueOp(pTrans, sysValId, value)) != NDBT_OK) {
     ERR(pTrans->getNdbError());
     m_ndb.closeTransaction(pTrans);
-    return NDBT_FAILED;
+    return result;
   }
 
   check = pTrans->execute(Commit);
   if( check == -1 ) {
     ERR(pTrans->getNdbError());
+    if(pTrans->getNdbError().status == NdbError::TemporaryError)
+    {
+      m_ndb.closeTransaction(pTrans);
+      return NDBT_TEMPORARY;
+    }
     m_ndb.closeTransaction(pTrans);
     return NDBT_FAILED;
   }
@@ -2099,6 +2089,8 @@ int Bank::increaseSystemValue(SystemValueId sysValId, Uint64 &value){
   NdbConnection* pTrans = m_ndb.startTransaction();
   if (pTrans == NULL){
     ERR(m_ndb.getNdbError());
+    if (m_ndb.getNdbError().status == NdbError::TemporaryError)
+      DBUG_RETURN(NDBT_TEMPORARY);
     DBUG_RETURN(NDBT_FAILED);
   }
     
@@ -2134,6 +2126,11 @@ int Bank::increaseSystemValue(SystemValueId sysValId, Uint64 &value){
   check = pTrans->execute(NoCommit);
   if( check == -1 ) {
     ERR(pTrans->getNdbError());
+    if (pTrans->getNdbError().status == NdbError::TemporaryError)
+    {
+      m_ndb.closeTransaction(pTrans);
+      DBUG_RETURN(NDBT_TEMPORARY);
+    }
     m_ndb.closeTransaction(pTrans);
     DBUG_RETURN(NDBT_FAILED);
   }
@@ -2208,6 +2205,11 @@ int Bank::increaseSystemValue(SystemValueId sysValId, Uint64 &value){
   check = pTrans->execute(Commit);
   if( check == -1 ) {
     ERR(pTrans->getNdbError());
+    if (pTrans->getNdbError().status == NdbError::TemporaryError)
+    {
+      m_ndb.closeTransaction(pTrans);
+      DBUG_RETURN(NDBT_TEMPORARY);
+    }
     m_ndb.closeTransaction(pTrans);
     DBUG_RETURN(NDBT_FAILED);
   }
@@ -2242,6 +2244,8 @@ int Bank::increaseSystemValue2(SystemValueId sysValId, Uint64 &value){
   NdbConnection* pTrans = m_ndb.startTransaction();
   if (pTrans == NULL){
     ERR(m_ndb.getNdbError());
+    if(m_ndb.getNdbError().status == NdbError::TemporaryError)
+      return NDBT_TEMPORARY;
     return NDBT_FAILED;
   }
     
@@ -2284,6 +2288,11 @@ int Bank::increaseSystemValue2(SystemValueId sysValId, Uint64 &value){
   check = pTrans->execute(Commit);
   if( check == -1 ) {
     ERR(pTrans->getNdbError());
+    if(pTrans->getNdbError().status == NdbError::TemporaryError)
+    {
+      m_ndb.closeTransaction(pTrans);
+      return NDBT_TEMPORARY;
+    }
     m_ndb.closeTransaction(pTrans);
     return NDBT_FAILED;
   }
@@ -2308,16 +2317,11 @@ int Bank::prepareGetCurrTimeOp(NdbConnection *pTrans, Uint64 &time){
 
 
 int Bank::performSumAccounts(int maxSleepBetweenSums, int yield){
-  if (init() != NDBT_OK)
-    return NDBT_FAILED;
   
   int yieldCounter = 0;
 
   while (1){
 
-    while (m_ndb.waitUntilReady(10) != 0)
-      ndbout << "Waiting for ndb to be ready" << endl;
-
     Uint32 sumAccounts = 0;
     Uint32 numAccounts = 0;
     if (getSumAccounts(sumAccounts, numAccounts) != NDBT_OK){

ndb/test/ndbapi/bank/Bank.hpp

@@ -27,7 +27,7 @@
 class Bank {
 public:  
 
-  Bank();
+  Bank(bool init = true);
 
   int createAndLoadBank(bool overWrite, int num_accounts=10);
   int dropBank();

ndb/test/ndbapi/testSRBank.cpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include <NDBT.hpp>
+#include <NDBT_Test.hpp>
+#include <HugoTransactions.hpp>
+#include <UtilTransactions.hpp>
+#include <NdbBackup.hpp>
+
+#include "bank/Bank.hpp"
+
+int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){
+  Bank bank;
+  int overWriteExisting = true;
+  if (bank.createAndLoadBank(overWriteExisting, 10) != NDBT_OK)
+    return NDBT_FAILED;
+  return NDBT_OK;
+}
+
+/**
+ *
+ * SR 0 - normal
+ * SR 1 - shutdown in progress
+ * SR 2 - restart in progress
+ */
+int runBankTimer(NDBT_Context* ctx, NDBT_Step* step){
+  int wait = 5; // Max seconds between each "day"
+  int yield = 1; // Loops before bank returns 
+
+  ctx->incProperty("ThreadCount");
+  while (!ctx->isTestStopped()) 
+  {
+    Bank bank;
+    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+      if(bank.performIncreaseTime(wait, yield) == NDBT_FAILED)
+	break;
+
+    ndbout_c("runBankTimer is stopped");
+    ctx->incProperty("ThreadStopped");
+    if(ctx->getPropertyWait("SR", (Uint32)0))
+      break;
+  }
+  return NDBT_OK;
+}
+
+int runBankTransactions(NDBT_Context* ctx, NDBT_Step* step){
+  int wait = 0; // Max ms between each transaction
+  int yield = 1; // Loops before bank returns 
+
+  ctx->incProperty("ThreadCount");
+  while (!ctx->isTestStopped()) 
+  {
+    Bank bank;
+    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+      if(bank.performTransactions(0, 1) == NDBT_FAILED)
+	break;
+    
+    ndbout_c("runBankTransactions is stopped");
+    ctx->incProperty("ThreadStopped");
+    if(ctx->getPropertyWait("SR", (Uint32)0))
+      break;
+  }
+  return NDBT_OK;
+}
+
+int runBankGL(NDBT_Context* ctx, NDBT_Step* step){
+  int yield = 1; // Loops before bank returns 
+  int result = NDBT_OK;
+
+  ctx->incProperty("ThreadCount");
+  while (ctx->isTestStopped() == false) 
+  {
+    Bank bank;
+    while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1)
+      if (bank.performMakeGLs(yield) != NDBT_OK)
+      {
+	if(ctx->getProperty("SR") != 0)
+	  break;
+	ndbout << "bank.performMakeGLs FAILED" << endl;
+	return NDBT_FAILED;
+      }
+    
+    ndbout_c("runBankGL is stopped");
+    ctx->incProperty("ThreadStopped");
+    if(ctx->getPropertyWait("SR", (Uint32)0))
+      break;
+  }
+  return NDBT_OK;
+}
+
+int runBankSum(NDBT_Context* ctx, NDBT_Step* step){
+  Bank bank;
+  int wait = 2000; // Max ms between each sum of accounts
+  int yield = 1; // Loops before bank returns 
+  int result = NDBT_OK;
+
+  while (ctx->isTestStopped() == false) {
+    if (bank.performSumAccounts(wait, yield) != NDBT_OK){
+      ndbout << "bank.performSumAccounts FAILED" << endl;
+      result = NDBT_FAILED;
+    }
+  }
+  return result ;
+}
+
+#define CHECK(b) if (!(b)) { \
+  g_err << "ERR: "<< step->getName() \
+         << " failed on line " << __LINE__ << endl; \
+  result = NDBT_FAILED; \
+  continue; } 
+
+int runSR(NDBT_Context* ctx, NDBT_Step* step)
+{
+  int result = NDBT_OK;
+  int runtime = ctx->getNumLoops();
+  int sleeptime = ctx->getNumRecords();
+  NdbRestarter restarter;
+  bool abort = true;
+  int timeout = 180;
+
+  Uint32 now;
+  const Uint32 stop = time(0)+ runtime;
+  while(!ctx->isTestStopped() && ((now= time(0)) < stop) && result == NDBT_OK)
+  {
+    ndbout << " -- Sleep " << sleeptime << "s " << endl;
+    NdbSleep_SecSleep(sleeptime);
+    ndbout << " -- Shutting down " << endl;
+    ctx->setProperty("SR", 1);
+    CHECK(restarter.restartAll(false, true, abort) == 0);
+    ctx->setProperty("SR", 2);
+    CHECK(restarter.waitClusterNoStart(timeout) == 0);
+
+    Uint32 cnt = ctx->getProperty("ThreadCount");
+    Uint32 curr= ctx->getProperty("ThreadStopped");
+    while(curr != cnt)
+    {
+      ndbout_c("%d %d", curr, cnt);
+      NdbSleep_MilliSleep(100);
+      curr= ctx->getProperty("ThreadStopped");
+    }
+
+    ctx->setProperty("ThreadStopped", (Uint32)0);
+    CHECK(restarter.startAll() == 0);
+    CHECK(restarter.waitClusterStarted(timeout) == 0);
+    
+    ndbout << " -- Validating starts " << endl;
+    {
+      int wait = 0;
+      int yield = 1;
+      Bank bank;
+      if (bank.performSumAccounts(wait, yield) != 0)
+      {
+	ndbout << "bank.performSumAccounts FAILED" << endl;
+	return NDBT_FAILED;
+      }
+
+      if (bank.performValidateAllGLs() != 0)
+      {
+	ndbout << "bank.performValidateAllGLs FAILED" << endl;
+	return NDBT_FAILED;
+      }
+    }
+
+    ndbout << " -- Validating complete " << endl;
+    ctx->setProperty("SR", (Uint32)0);
+    ctx->broadcast();
+  }
+  ctx->stopTest();
+  return NDBT_OK;
+}
+
+int runDropBank(NDBT_Context* ctx, NDBT_Step* step){
+  Bank bank;
+  if (bank.dropBank() != NDBT_OK)
+    return NDBT_FAILED;
+  return NDBT_OK;
+}
+
+
+NDBT_TESTSUITE(testSRBank);
+TESTCASE("Graceful", 
+	 " Test that a consistent bank is restored after graceful shutdown\n"
+	 "1.  Create bank\n"
+	 "2.  Start bank and let it run\n"
+	 "3.  Restart ndb and verify consistency\n"
+	 "4.  Drop bank\n")
+{
+  INITIALIZER(runCreateBank);
+  STEP(runBankTimer);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankGL);
+  STEP(runSR);
+}
+TESTCASE("Abort", 
+	 " Test that a consistent bank is restored after graceful shutdown\n"
+	 "1.  Create bank\n"
+	 "2.  Start bank and let it run\n"
+	 "3.  Restart ndb and verify consistency\n"
+	 "4.  Drop bank\n")
+{
+  INITIALIZER(runCreateBank);
+  STEP(runBankTimer);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankTransactions);
+  STEP(runBankGL);
+  STEP(runSR);
+  FINALIZER(runDropBank);
+}
+NDBT_TESTSUITE_END(testSRBank);
+
+int main(int argc, const char** argv){
+  ndb_init();
+  return testSRBank.execute(argc, argv);
+}
+
+

ndb/test/src/NDBT_Test.cpp

@@ -145,6 +145,15 @@ NDBT_Context::decProperty(const char * name){
   NdbCondition_Broadcast(propertyCondPtr);
   NdbMutex_Unlock(propertyMutexPtr);
 }
+void
+NDBT_Context::incProperty(const char * name){
+  NdbMutex_Lock(propertyMutexPtr);
+  Uint32 val = 0;
+  props.get(name, &val);
+  props.put(name, (val + 1), true);
+  NdbCondition_Broadcast(propertyCondPtr);
+  NdbMutex_Unlock(propertyMutexPtr);
+}
 
 void  NDBT_Context::setProperty(const char* _name, const char* _val){ 
   NdbMutex_Lock(propertyMutexPtr);

ndb/test/src/UtilTransactions.cpp

@@ -766,19 +766,29 @@ UtilTransactions::selectCount(Ndb* pNdb,
   int                  check;
   NdbScanOperation     *pOp;
 
-  if(!pTrans)
-    pTrans = pNdb->startTransaction();
   while (true){
-
     if (retryAttempt >= retryMax){
       g_info << "ERROR: has retried this operation " << retryAttempt 
 	     << " times, failing!" << endl;
       return NDBT_FAILED;
     }
+    if(!pTrans)
+      pTrans = pNdb->startTransaction();
+    
+    if(!pTrans)
+    {
+      const NdbError err = pNdb->getNdbError();
+      
+      if (err.status == NdbError::TemporaryError)
+	continue;
+      return NDBT_FAILED;
+    }
+    
     pOp = pTrans->getNdbScanOperation(tab.getName());	
     if (pOp == NULL) {
       ERR(pTrans->getNdbError());
       pNdb->closeTransaction(pTrans);
+      pTrans = 0;
       return NDBT_FAILED;
     }
 
@@ -786,6 +796,7 @@ UtilTransactions::selectCount(Ndb* pNdb,
     if( rs == 0) {
       ERR(pTrans->getNdbError());
       pNdb->closeTransaction(pTrans);
+      pTrans = 0;
       return NDBT_FAILED;
     }
 
@@ -799,6 +810,7 @@ UtilTransactions::selectCount(Ndb* pNdb,
       if( check == -1 ) {
 	ERR(pTrans->getNdbError());
 	pNdb->closeTransaction(pTrans);
+	pTrans = 0;
 	return NDBT_FAILED;
       }
     }
@@ -808,6 +820,7 @@ UtilTransactions::selectCount(Ndb* pNdb,
     if( check == -1 ) {
       ERR(pTrans->getNdbError());
       pNdb->closeTransaction(pTrans);
+      pTrans = 0;
       return NDBT_FAILED;
     }
 
@@ -823,16 +836,19 @@ UtilTransactions::selectCount(Ndb* pNdb,
       
       if (err.status == NdbError::TemporaryError){
 	pNdb->closeTransaction(pTrans);
+	pTrans = 0;
 	NdbSleep_MilliSleep(50);
 	retryAttempt++;
 	continue;
       }
       ERR(err);
       pNdb->closeTransaction(pTrans);
+      pTrans = 0;
       return NDBT_FAILED;
     }
     
     pNdb->closeTransaction(pTrans);
+    pTrans = 0;
     
     if (count_rows != NULL){
       *count_rows = rows;