BUG#29674 Restore/backup are endian compatible in 5.0

ndb/src/ndbapi/NdbDictionaryImpl.cpp:
  Twiddle the "replicaCount" and "fragCount" variable when restore data from different endian.
ndb/src/ndbapi/NdbDictionaryImpl.hpp:
  Add byte order variable
ndb/tools/restore/Restore.cpp:
  Twiddle blob, datatime,timestamp when do restore in different endian.
mysql-test/r/ndb_restore_different_endian_data.result:
  Test case result for restore data from different endian
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1-0.1.Data:
  Test case data
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1-0.2.Data:
  Test case data
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1.1.ctl:
  Test case data
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1.1.log:
  Test case data
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1.2.ctl:
  Test case data
mysql-test/std_data/ndb_backup50_data_be/BACKUP-1.2.log:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1-0.1.Data:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1-0.2.Data:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1.1.ctl:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1.1.log:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1.2.ctl:
  Test case data
mysql-test/std_data/ndb_backup50_data_le/BACKUP-1.2.log:
  Test case data
mysql-test/t/ndb_restore_different_endian_data.test:
  Test case for restore data from different endian

mysql-test/t/ndb_restore_different_endian_data.test

+-- source include/have_ndb.inc
+-- source include/ndb_default_cluster.inc
+-- source include/not_embedded.inc
+
+#
+# Bug #27543 restore of backup from different endian does not work for blob column
+# Bug #30024 restore of backup from different endian does not work for datetime column
+# Bug #28674 backup will run forever if disk full and later write succes will kill ndb node
+#
+# The table structure and data list below
+#
+# CREATE TABLE  t_num (
+#   t_pk		INT PRIMARY KEY,
+#   t_bit		BIT(64), 
+#   t_tinyint	TINYINT, 
+#   t_bool	BOOL, 
+#   t_smallint	SMALLINT,
+#   t_mediumint	MEDIUMINT,
+#   t_int		INT,
+#   t_bigint	BIGINT,
+#   t_float	FLOAT,
+#   t_double	DOUBLE,
+#   t_decimal	DECIMAL (37, 16)
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_num VALUE (
+#   1,
+#   b'1010101010101010101010101010101010101010101010101010101010101010',
+#   125,
+#   1,
+#   32765,
+#   8388606,
+#   2147483647,
+#   9223372036854775807,
+#   1e+20,
+#   1e+150,
+#   '331.0000000000'
+# );
+# 
+# CREATE TABLE  t_datetime (
+#   t_pk		INT PRIMARY KEY,
+#   t_date	DATE,
+#   t_datetime	DATETIME,
+#   t_timestamp	TIMESTAMP,
+#   t_time	TIME,
+#   t_year	YEAR
+# ) ENGINE=NDBCLUSTER;
+#   
+# INSERT INTO t_datetime VALUE (
+#   1,
+#   '1998-01-01',
+#   '2006-08-10 10:11:12',
+#   20021029165106,
+#   '19:38:34',
+#   2155
+# );
+# 
+# CREATE TABLE  t_string_1 (
+#   t_pk		INT PRIMARY KEY,
+#   t_char	CHAR(255),
+#   t_varchar	VARCHAR(655),
+#   t_binary	BINARY(255),
+#   t_varbinary	VARBINARY(6553)
+# ) ENGINE=NDBCLUSTER;
+# 
+# CREATE TABLE  t_string_2 (
+#   t_pk		INT PRIMARY KEY,
+#   t_tinyblob	TINYBLOB,
+#   t_tinytext	TINYTEXT,
+#   t_blob	BLOB,
+#   t_text	TEXT,
+#   t_mediumblob	MEDIUMBLOB,
+#   t_mediumtext	MEDIUMTEXT,
+#   t_longblob	LONGBLOB,
+#   t_longtext	LONGTEXT,
+#   t_enum	ENUM('001001','001004','001010','001018','001019','001020','001021','001027','001028','001029','001030','001031','001100','002003','002004','002005','002007')DEFAULT '001001' NOT NULL,
+#   t_set		SET('a','B')
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_string_1 VALUE (
+#   1,
+#   'abcdefghijklmn',
+#   'abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn',
+#   0x612020,
+#   0x4100
+# );
+# 
+# INSERT INTO t_string_2 VALUE (
+#   1,
+#   'abcdefghijklmnabcdefghijklmn',
+#   'abcdefghijklmnabcdefghijklmn',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   '001001',
+#   'a'
+# ); 
+#  
+# CREATE TABLE t_gis (
+#   t_pk			INT PRIMARY KEY,
+#   t_point		POINT,
+#   t_linestring		LINESTRING,
+#   t_polygon		POLYGON,
+#   t_multipoint		MULTIPOINT,
+#   t_multilinestring	MULTILINESTRING,
+#   t_multipolygon	MULTIPOLYGON,
+#   t_geometrycollection	GEOMETRYCOLLECTION,
+#   t_geometry		GEOMETRY
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_gis VALUE (
+#   1,
+#   PointFromText('POINT(10 10)'),
+#   LineStringFromText('LINESTRING(10 10,20 10,20 20,10 20,10 10)'),
+#   PolyFromText('POLYGON((0 0,50 0,50 50,0 50,0 0), (10 10,20 10,20 20,10 20,10 10))'),
+#   MPointFromText('MULTIPOINT(1 1,11 11,11 21,21 21)'),
+#   MLineFromText('MULTILINESTRING((10 48,10 21,10 0))'),
+#   MPolyFromText('MULTIPOLYGON(((28 26,28 0,84 0,84 42,28 26),(52 18,66 23,73 9,48 6,52 18)),((59 18,67 18,67 13,59 13,59 18)))'),
+#   GeomCollFromText('GEOMETRYCOLLECTION(POINT(0 0), LINESTRING(0 0,10 10))'),
+#   MPolyFromText('MULTIPOLYGON(((28 26,28 0,84 0,84 42,28 26),(52 18,66 23,73 9,48 6,52 18)),((59 18,67 18,67 13,59 13,59 18)))')
+# );
+# 
+# INSERT INTO t_gis VALUE (
+#   2,
+#   PointFromWKB(AsWKB(PointFromText('POINT(10 20)'))),
+#   LineStringFromWKB(LineString(Point(10, 10), Point(40, 10))),
+#   PolyFromWKB(Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0)))),
+#   MPointFromWKB(MultiPoint(Point(3, 6), Point(4, 10))),
+#   MLineFromWKB(MultiLineString(LineString(Point(1, 2), Point(3, 5)), LineString(Point(2, 5), Point(5, 8), Point(21, 7)))),
+#   MPolyFromWKB(MultiPolygon(Polygon(LineString(Point(0, 3), Point(3, 3), Point(3, 0), Point(0, 3))))),
+#   GeometryFromWKB(GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9)))),
+#   GeometryFromWKB(GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9))))
+# );
+
+
+
+#
+# Restore backup files (from little endian)
+#
+
+--disable_warnings
+USE test;
+DROP TABLE IF EXISTS t_num,t_datetime,t_string_1,t_string_2,t_gis;
+--enable_warnings
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 1 -m -r $MYSQL_TEST_DIR/std_data/ndb_backup50_data_le >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 2 -r $MYSQL_TEST_DIR/std_data/ndb_backup50_data_le >> $NDB_TOOLS_OUTPUT
+SHOW TABLES;
+SHOW CREATE TABLE t_num;
+SHOW CREATE TABLE t_datetime;
+SHOW CREATE TABLE t_string_1;
+SHOW CREATE TABLE t_string_2;
+SHOW CREATE TABLE t_gis;
+SELECT * FROM t_datetime;
+SELECT t_pk,hex(t_bit),t_tinyint,t_bool,t_smallint,t_mediumint,t_int,t_bigint,t_float,t_double,t_decimal FROM t_num;
+SELECT t_pk,t_char,t_varchar,hex(t_binary),hex(t_varbinary) FROM t_string_1;
+SELECT * FROM t_string_2;
+SELECT AsText(t_point), AsText(t_linestring),AsText(t_polygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_multipoint), AsText(t_multilinestring),AsText(t_multipolygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_geometrycollection), AsText(t_geometry) FROM t_gis ORDER by t_pk;
+
+#
+# Restore backup files (from big endian)
+#
+
+DROP TABLE t_num,t_datetime,t_string_1,t_string_2,t_gis;
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 1 -m -r $MYSQL_TEST_DIR/std_data/ndb_backup50_data_be >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 2 -r $MYSQL_TEST_DIR/std_data/ndb_backup50_data_be >> $NDB_TOOLS_OUTPUT
+SHOW TABLES;
+SHOW CREATE TABLE t_num;
+SHOW CREATE TABLE t_datetime;
+SHOW CREATE TABLE t_string_1;
+SHOW CREATE TABLE t_string_2;
+SHOW CREATE TABLE t_gis;
+SELECT * FROM t_datetime;
+SELECT t_pk,hex(t_bit),t_tinyint,t_bool,t_smallint,t_mediumint,t_int,t_bigint,t_float,t_double,t_decimal FROM t_num;
+SELECT t_pk,t_char,t_varchar,hex(t_binary),hex(t_varbinary) FROM t_string_1;
+SELECT * FROM t_string_2;
+SELECT AsText(t_point), AsText(t_linestring),AsText(t_polygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_multipoint), AsText(t_multilinestring),AsText(t_multipolygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_geometrycollection), AsText(t_geometry) FROM t_gis ORDER by t_pk;
+
+DROP TABLE t_num,t_datetime,t_string_1,t_string_2,t_gis;

ndb/src/ndbapi/NdbDictionaryImpl.cpp

@@ -1220,7 +1220,8 @@ indexTypeMapping[] = {
 int
 NdbDictInterface::parseTableInfo(NdbTableImpl ** ret,
 				 const Uint32 * data, Uint32 len,
-				 bool fullyQualifiedNames)
+				 bool fullyQualifiedNames,
+				 bool hostByteOrder)
 {
   DBUG_ENTER("NdbDictInterface::parseTableInfo");
 
@@ -1379,8 +1380,14 @@ NdbDictInterface::parseTableInfo(NdbTableImpl ** ret,
 
   if(tableDesc.FragmentDataLen > 0)
   {
-    Uint32 replicaCount = tableDesc.FragmentData[0];
-    Uint32 fragCount = tableDesc.FragmentData[1];
+    Uint16 replicaCount = tableDesc.FragmentData[0];
+    Uint16 fragCount = tableDesc.FragmentData[1];
+
+    if(hostByteOrder == false)
+    {
+      replicaCount = ((replicaCount & 0xFF00) >> 8) |((replicaCount & 0x00FF) << 8);
+      fragCount = ((fragCount & 0xFF00) >> 8) |((fragCount & 0x00FF) << 8);
+    }
 
     impl->m_replicaCount = replicaCount;
     impl->m_fragmentCount = fragCount;

ndb/src/ndbapi/NdbDictionaryImpl.hpp

@@ -262,7 +262,8 @@ public:
 
   static int parseTableInfo(NdbTableImpl ** dst, 
 			    const Uint32 * data, Uint32 len,
-			    bool fullyQualifiedNames);
+			    bool fullyQualifiedNames,
+			    bool hostByteOrder = true);
   
   static int create_index_obj_from_table(NdbIndexImpl ** dst, 
 					 NdbTableImpl* index_table,

ndb/tools/restore/Restore.cpp

@@ -324,7 +324,11 @@ bool
 RestoreMetaData::parseTableDescriptor(const Uint32 * data, Uint32 len)
 {
   NdbTableImpl* tableImpl = 0;
-  int ret = NdbDictInterface::parseTableInfo(&tableImpl, data, len, false);
+  int ret = 0;
+  if(!m_hostByteOrder)
+      ret = NdbDictInterface::parseTableInfo(&tableImpl, data, len, false, false);
+  else
+      ret = NdbDictInterface::parseTableInfo(&tableImpl, data, len, false);
 
   if (ret != 0) {
     err << "parseTableInfo " << " failed" << endl;
@@ -478,6 +482,10 @@ RestoreDataIterator::getNextTuple(int  & res)
     attr_data->void_value = ptr;
     attr_data->size = 4*sz;
 
+    if(!m_hostByteOrder
+        && attr_desc->m_column->getType() == NdbDictionary::Column::Timestamp)
+      attr_data->u_int32_value[0] = Twiddle32(attr_data->u_int32_value[0]);
+
     if(!Twiddle(attr_desc, attr_data))
       {
 	res = -1;
@@ -520,6 +528,29 @@ RestoreDataIterator::getNextTuple(int  & res)
      */
     const Uint32 arraySize = (4 * sz) / (attr_desc->size / 8);
     assert(arraySize >= attr_desc->arraySize);
+    
+    //convert the length of blob(v1) and text(v1)
+    if(!m_hostByteOrder
+        && (attr_desc->m_column->getType() == NdbDictionary::Column::Blob
+           || attr_desc->m_column->getType() == NdbDictionary::Column::Text))
+    {
+      char* p = (char*)&attr_data->u_int64_value[0];
+      Uint64 x;
+      memcpy(&x, p, sizeof(Uint64));
+      x = Twiddle64(x);
+      memcpy(p, &x, sizeof(Uint64));
+    }
+
+    if(!m_hostByteOrder
+        && attr_desc->m_column->getType() == NdbDictionary::Column::Datetime)
+    {
+      char* p = (char*)&attr_data->u_int64_value[0];
+      Uint64 x;
+      memcpy(&x, p, sizeof(Uint64));
+      x = Twiddle64(x);
+      memcpy(p, &x, sizeof(Uint64));
+    }
+
     if(!Twiddle(attr_desc, attr_data, attr_desc->arraySize))
       {
 	res = -1;