re-enginered ndb restore to remove new/deletes and data copy

ndb/src/kernel/blocks/backup/restore/Makefile.am

 
 ndbtools_PROGRAMS = ndb_restore
 
-ndb_restore_SOURCES = main.cpp Restore.cpp
+ndb_restore_SOURCES = main.cpp consumer.cpp consumer_restore.cpp consumer_printer.cpp Restore.cpp
 
 LDADD_LOC = $(top_builddir)/ndb/src/libndbclient.la
 

ndb/src/kernel/blocks/backup/restore/Makefile_old

-include .defs.mk
-
-TYPE := *
-
-BIN_TARGET := restore
-BIN_TARGET_LIBS := 
-BIN_TARGET_ARCHIVES := NDB_API
-
-CCFLAGS_LOC = -I.. -I$(NDB_TOP)/src/ndbapi  -I$(NDB_TOP)/include/ndbapi -I$(NDB_TOP)/include/util  -I$(NDB_TOP)/include/portlib -I$(NDB_TOP)/include/kernel
-
-#ifneq ($(MYSQLCLUSTER_TOP),)
-#CCFLAGS_LOC +=-I$(MYSQLCLUSTER_TOP)/include -D USE_MYSQL
-#LDFLAGS_LOC += -L$(MYSQLCLUSTER_TOP)/libmysql_r/ -lmysqlclient_r
-#endif
-
-SOURCES = main.cpp Restore.cpp
-
-include $(NDB_TOP)/Epilogue.mk
-
-

ndb/src/kernel/blocks/backup/restore/Restore.hpp

@@ -18,6 +18,7 @@
 #define RESTORE_H
 
 #include <ndb_global.h>
+#include <NdbOut.hpp>
 #include <BackupFormat.hpp>
 #include <NdbApi.hpp>
 #include "myVector.hpp"
@@ -25,6 +26,8 @@
 #include <ndb_version.h>
 #include <version.h>
 
+static const char * delimiter = ";"; // Delimiter in file dump
+
 const int FileNameLenC = 256;
 const int TableNameLenC = 256;
 const int AttrNameLenC = 256;
@@ -82,26 +85,30 @@ public:
 
 struct AttributeS {
   const AttributeDesc * Desc;
-  AttributeData Data;
+  AttributeData * Data;
 };
 
 class TupleS {
 private:
   friend class RestoreDataIterator;
   
-  const TableS * m_currentTable;
-  myVector<AttributeS*> allAttributes;
-  Uint32 * dataRecord;
+  const TableS *m_currentTable;
+  AttributeData *allAttrData;
   bool prepareRecord(const TableS &);
   
 public:
-  TupleS() {dataRecord = NULL;};
-  ~TupleS() {if(dataRecord != NULL) delete [] dataRecord;};
-  int getNoOfAttributes() const { return allAttributes.size(); };  
-  const TableS * getTable() const { return m_currentTable;};
-  const AttributeS * operator[](int i) const { return allAttributes[i];};
-  Uint32 * getDataRecord() { return dataRecord;};
-  void createDataRecord(Uint32 bytes) { dataRecord = new Uint32[bytes];};
+  TupleS() {};
+  ~TupleS()
+  {
+    if (allAttrData)
+      delete [] allAttrData;
+  };
+  TupleS(const TupleS& tuple); // disable copy constructor
+  TupleS & operator=(const TupleS& tuple);
+  int getNoOfAttributes() const;
+  const TableS * getTable() const;
+  const AttributeDesc * getDesc(int i) const;
+  AttributeData * getData(int i) const;
 }; // class TupleS
 
 class TableS {
@@ -206,7 +213,7 @@ public:
   const char * getFilename() const { return m_fileName;}
   Uint32 getNodeId() const { return m_nodeId;}
   const BackupFormat::FileHeader & getFileHeader() const { return m_fileHeader;}
-  bool Twiddle(AttributeS * attr, Uint32 arraySize = 0);
+  bool Twiddle(const AttributeDesc *  attr_desc, AttributeData * attr_data, Uint32 arraySize = 0);
 };
 
 class RestoreMetaData : public BackupFile {
@@ -243,20 +250,28 @@ public:
 
 class RestoreDataIterator : public BackupFile {
   const RestoreMetaData & m_metaData;
-
   Uint32 m_count;
-  TupleS  m_tuple;
   const TableS* m_currentTable;
+  TupleS m_tuple;
+
+  void * m_buffer;
+  void * m_buffer_ptr;
+  Uint32 m_buffer_sz;
+  Uint32 m_buffer_data_left;
+  void (* free_data_callback)();
 public:
 
   // Constructor
-  RestoreDataIterator(const RestoreMetaData &);
+  RestoreDataIterator(const RestoreMetaData &, void (* free_data_callback)());
   ~RestoreDataIterator();
   
   // Read data file fragment header
   bool readFragmentHeader(int & res);
   bool validateFragmentFooter();
 
+  Uint32 get_buffer_ptr(void **p_buf_ptr, Uint32 size, Uint32 nmemb, FILE *stream);
+  Uint32 fread_buffer(void *ptr, Uint32 size, Uint32 nmemb, FILE *stream);
+
   const TupleS *getNextTuple(int & res);
 };
 
@@ -286,6 +301,11 @@ public:
   const LogEntry * getNextLogEntry(int & res);
 };
 
+NdbOut& operator<<(NdbOut& ndbout, const TableS&);
+NdbOut& operator<<(NdbOut& ndbout, const TupleS&);
+NdbOut& operator<<(NdbOut& ndbout, const LogEntry&);
+NdbOut& operator<<(NdbOut& ndbout, const RestoreMetaData&);
+
 #endif
 
 

ndb/src/kernel/blocks/backup/restore/consumer.cpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "consumer.hpp"
+
+#ifdef USE_MYSQL
+int
+BackupConsumer::create_table_string(const TableS & table,
+				    char * tableName,
+				    char *buf){
+  int pos = 0;
+  int pos2 = 0;
+  char buf2[2048];
+
+  pos += sprintf(buf+pos, "%s%s", "CREATE TABLE ",  tableName);
+  pos += sprintf(buf+pos, "%s", "(");
+  pos2 += sprintf(buf2+pos2, "%s", " primary key(");
+
+  for (int j = 0; j < table.getNoOfAttributes(); j++) 
+  {
+    const AttributeDesc * desc = table[j];
+    //   ndbout << desc->name << ": ";
+    pos += sprintf(buf+pos, "%s%s", desc->m_column->getName()," ");
+    switch(desc->m_column->getType()){
+    case NdbDictionary::Column::Int:
+      pos += sprintf(buf+pos, "%s", "int");
+      break;
+    case NdbDictionary::Column::Unsigned:
+      pos += sprintf(buf+pos, "%s", "int unsigned");
+      break;
+    case NdbDictionary::Column::Float:
+      pos += sprintf(buf+pos, "%s", "float");
+      break;
+    case NdbDictionary::Column::Decimal:
+      pos += sprintf(buf+pos, "%s", "decimal");
+      break;
+    case NdbDictionary::Column::Char:
+      pos += sprintf(buf+pos, "%s", "char");
+      break;
+    case NdbDictionary::Column::Varchar:
+      pos += sprintf(buf+pos, "%s", "varchar");
+      break;
+    case NdbDictionary::Column::Binary:
+      pos += sprintf(buf+pos, "%s", "binary");
+      break;
+    case NdbDictionary::Column::Varbinary:
+      pos += sprintf(buf+pos, "%s", "varchar binary");
+      break;
+    case NdbDictionary::Column::Bigint:
+      pos += sprintf(buf+pos, "%s", "bigint");
+      break;
+    case NdbDictionary::Column::Bigunsigned:
+      pos += sprintf(buf+pos, "%s", "bigint unsigned");
+      break;
+    case NdbDictionary::Column::Double:
+      pos += sprintf(buf+pos, "%s", "double");
+      break;
+    case NdbDictionary::Column::Datetime:
+      pos += sprintf(buf+pos, "%s", "datetime");
+      break;
+    case NdbDictionary::Column::Timespec:
+      pos += sprintf(buf+pos, "%s", "time");
+      break;
+    case NdbDictionary::Column::Undefined:
+      //      pos += sprintf(buf+pos, "%s", "varchar binary");
+      return -1;
+      break;
+    default:
+      //pos += sprintf(buf+pos, "%s", "varchar binary");
+      return -1;
+    }
+    if (desc->arraySize > 1) {
+      int attrSize = desc->arraySize;
+      pos += sprintf(buf+pos, "%s%u%s",
+		     "(",
+		     attrSize,
+		     ")");
+    }
+    if (desc->m_column->getPrimaryKey()) {
+      pos += sprintf(buf+pos, "%s", " not null");
+      pos2 += sprintf(buf2+pos2, "%s%s", desc->m_column->getName(), ",");
+    }
+    pos += sprintf(buf+pos, "%s", ",");
+  } // for
+  pos2--; // remove trailing comma
+  pos2 += sprintf(buf2+pos2, "%s", ")");
+  //  pos--; // remove trailing comma
+
+  pos += sprintf(buf+pos, "%s", buf2);
+  pos += sprintf(buf+pos, "%s", ") type=ndbcluster");
+  return 0;
+}
+
+#endif // USE_MYSQL

ndb/src/kernel/blocks/backup/restore/consumer.hpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef CONSUMER_HPP
+#define CONSUMER_HPP
+
+#include "Restore.hpp"
+
+class BackupConsumer {
+public:
+  virtual bool init() { return true;}
+  virtual bool table(const TableS &){return true;}
+#ifdef USE_MYSQL
+  virtual bool table(const TableS &, MYSQL* mysqlp) {return true;};
+#endif
+  virtual void tuple(const TupleS &){}
+  virtual void tuple_free(){}
+  virtual void endOfTuples(){}
+  virtual void logEntry(const LogEntry &){}
+  virtual void endOfLogEntrys(){}
+protected:
+#ifdef USE_MYSQL
+  int create_table_string(const TableS & table, char * ,char *);
+#endif
+};
+
+#endif

ndb/src/kernel/blocks/backup/restore/consumer_printer.cpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "consumer_printer.hpp"
+
+bool
+BackupPrinter::table(const TableS & tab)
+{
+  if (m_print || m_print_meta) 
+  {
+    m_ndbout << tab;
+    ndbout_c("Successfully printed table: %s", tab.m_dictTable->getName());
+  }
+  return true;
+}
+
+#ifdef USE_MYSQL
+bool
+BackupPrinter::table(const TableS & tab, MYSQL * mysql)
+{
+  if (m_print || m_print_meta) 
+  {
+    
+    char tmpTabName[MAX_TAB_NAME_SIZE*2];
+    sprintf(tmpTabName, "%s", tab.getTableName());
+    char * database = strtok(tmpTabName, "/");
+    char * schema   = strtok( NULL , "/");
+    char * tableName    = strtok( NULL , "/");
+    
+    /**
+     * this means that the user did not specify schema
+     * and it is a v2x backup
+     */
+    if(database == NULL)
+      return false;
+    if(schema == NULL)
+      return false;
+    if(tableName==NULL)
+      tableName = schema; 
+    
+    char stmtCreateDB[255];
+    
+    sprintf(stmtCreateDB,"CREATE DATABASE %s", database);
+    ndbout_c("%s", stmtCreateDB);
+    
+    
+    char buf [2048];
+    create_table_string(tab, tableName,  buf);
+    ndbout_c("%s", buf);
+    
+    ndbout_c("Successfully printed table: %s", tab.m_dictTable->getName());
+  }
+  return true;
+}
+
+#endif 
+
+void
+BackupPrinter::tuple(const TupleS & tup)
+{
+  m_dataCount++;  
+  if (m_print || m_print_data)
+    m_ndbout << tup << endl;  
+}
+
+void
+BackupPrinter::logEntry(const LogEntry & logE)
+{
+  if (m_print || m_print_log)
+    m_ndbout << logE << endl;
+  m_logCount++;
+}
+
+void
+BackupPrinter::endOfLogEntrys()
+{
+  if (m_print || m_print_log) 
+  {
+    ndbout << "Printed " << m_dataCount << " tuples and "
+	   << m_logCount << " log entries" 
+	   << " to stdout." << endl;
+  }
+}

ndb/src/kernel/blocks/backup/restore/consumer_printer.hpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef CONSUMER_PRINTER_HPP
+#define CONSUMER_PRINTER_HPP
+
+#include "consumer.hpp"
+
+class BackupPrinter : public BackupConsumer 
+{
+  NdbOut & m_ndbout;
+public:
+  BackupPrinter(NdbOut & out = ndbout) : m_ndbout(out) 
+  {
+    m_print = false;
+    m_print_log = false;
+    m_print_data = false;
+    m_print_meta = false;
+  }
+  
+  virtual bool table(const TableS &);
+#ifdef USE_MYSQL
+  virtual bool table(const TableS &, MYSQL* mysqlp);
+#endif
+  virtual void tuple(const TupleS &);
+  virtual void logEntry(const LogEntry &);
+  virtual void endOfTuples() {};
+  virtual void endOfLogEntrys();
+  bool m_print;
+  bool m_print_log;
+  bool m_print_data;
+  bool m_print_meta;
+  Uint32 m_logCount;
+  Uint32 m_dataCount;  
+};
+
+#endif

ndb/src/kernel/blocks/backup/restore/consumer_restore.hpp

+/* Copyright (C) 2003 MySQL AB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef CONSUMER_RESTORE_HPP
+#define CONSUMER_RESTORE_HPP
+
+#include "consumer.hpp"
+
+struct restore_callback_t {
+  class BackupRestore *restore;
+  class TupleS *tup;
+  class NdbConnection *connection;
+  int    retries;
+  restore_callback_t *next;
+};
+
+
+class BackupRestore : public BackupConsumer 
+{
+public:
+  BackupRestore(Uint32 parallelism=1) 
+  {
+    m_ndb = 0;
+    m_logCount = m_dataCount = 0;
+    m_restore = false;
+    m_restore_meta = false;
+    m_parallelism = parallelism;
+    m_callback = 0;
+    m_tuples = 0;
+    m_free_callback = 0;
+    m_transactions = 0;
+  }
+  
+  virtual ~BackupRestore();
+
+  virtual bool init();
+  virtual void release();
+  virtual bool table(const TableS &);
+#ifdef USE_MYSQL
+  virtual bool table(const TableS &, MYSQL* mysqlp);
+#endif
+  virtual void tuple(const TupleS &);
+  virtual void tuple_free();
+  virtual void tuple_a(restore_callback_t *cb);
+  virtual void cback(int result, restore_callback_t *cb);
+  virtual bool errorHandler(restore_callback_t *cb);
+  virtual void exitHandler();
+  virtual void endOfTuples();
+  virtual void logEntry(const LogEntry &);
+  virtual void endOfLogEntrys();
+  void connectToMysql();
+  Ndb * m_ndb;
+  bool m_restore;
+  bool m_restore_meta;
+  Uint32 m_logCount;
+  Uint32 m_dataCount;
+
+  Uint32 m_parallelism;
+  Uint32 m_transactions;
+
+  TupleS *m_tuples;
+  restore_callback_t *m_callback;
+  restore_callback_t *m_free_callback;
+};
+
+#endif