BUG#27543 Backup and restore can compatible with different endians for blob attribute

mysql-test/Makefile.am:
  Add install data for new test case
storage/ndb/tools/restore/Restore.cpp:
  Convert blob(text) length stored in main table
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1-0.1.Data:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1-0.2.Data:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1.1.ctl:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1.1.log:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1.2.ctl:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_be/BACKUP-1.2.log:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1-0.1.Data:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1-0.2.Data:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1.1.ctl:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1.1.log:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1.2.ctl:
  test data for endian test
mysql-test/std_data/ndb_backup51_data_le/BACKUP-1.2.log:
  test data for endian test
mysql-test/suite/ndb/r/ndb_restore_different_endian_data.result:
  test result for endian test
mysql-test/suite/ndb/t/ndb_restore_different_endian_data.test:
  test case for endian test

mysql-test/Makefile.am

@@ -43,6 +43,8 @@ dist-hook:
 		$(distdir)/std_data \
 		$(distdir)/std_data/ndb_backup50 \
 		$(distdir)/std_data/ndb_backup51 \
+		$(distdir)/std_data/ndb_backup51_data_be \
+		$(distdir)/std_data/ndb_backup51_data_le \
 		$(distdir)/lib
 	-$(INSTALL_DATA) $(srcdir)/t/*.def $(distdir)/t
 	$(INSTALL_DATA) $(srcdir)/t/*.test $(distdir)/t
@@ -68,6 +70,8 @@ dist-hook:
 	$(INSTALL_DATA) $(srcdir)/std_data/*.cnf $(distdir)/std_data
 	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup50/BACKUP* $(distdir)/std_data/ndb_backup50
 	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51/BACKUP* $(distdir)/std_data/ndb_backup51
+	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51_data_be/BACKUP* $(distdir)/std_data/ndb_backup51_data_be
+	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51_data_le/BACKUP* $(distdir)/std_data/ndb_backup51_data_le
 	$(INSTALL_DATA) $(srcdir)/lib/*.pl $(distdir)/lib
 	-rm -rf `find $(distdir)/suite -type d -name SCCS` $(distdir)/suite/row_lock
 
@@ -81,6 +85,8 @@ install-data-local:
 		$(DESTDIR)$(testdir)/std_data \
 		$(DESTDIR)$(testdir)/std_data/ndb_backup50 \
 		$(DESTDIR)$(testdir)/std_data/ndb_backup51 \
+		$(DESTDIR)$(testdir)/std_data/ndb_backup51_data_be \
+		$(DESTDIR)$(testdir)/std_data/ndb_backup51_data_le \
 		$(DESTDIR)$(testdir)/lib
 	$(INSTALL_DATA) $(srcdir)/README $(DESTDIR)$(testdir)
 	-$(INSTALL_DATA) $(srcdir)/t/*.def $(DESTDIR)$(testdir)/t
@@ -111,6 +117,8 @@ install-data-local:
 	$(INSTALL_DATA) $(srcdir)/std_data/*.cnf $(DESTDIR)$(testdir)/std_data
 	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup50/BACKUP* $(DESTDIR)$(testdir)/std_data/ndb_backup50
 	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51/BACKUP* $(DESTDIR)$(testdir)/std_data/ndb_backup51
+	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51_data_be/BACKUP* $(DESTDIR)$(testdir)/std_data/ndb_backup51_data_be
+	$(INSTALL_DATA) $(srcdir)/std_data/ndb_backup51_data_le/BACKUP* $(DESTDIR)$(testdir)/std_data/ndb_backup51_data_le
 	$(INSTALL_DATA) $(srcdir)/lib/*.pl $(DESTDIR)$(testdir)/lib
 	for f in `(cd $(srcdir); find suite -type f | egrep -v 'SCCS|row_lock')`; \
 	do \

mysql-test/suite/ndb/t/ndb_restore_different_endian_data.test

+-- source include/have_ndb.inc
+-- source include/ndb_default_cluster.inc
+-- source include/not_embedded.inc
+
+#
+# Bug #27543 restore of backup from different endian does not work for blob column
+# Bug #30024 restore of backup from different endian does not work for datetime column
+# Bug #28674 backup will run forever if disk full and later write succes will kill ndb node
+#
+# The table structure and data list below
+#
+# CREATE TABLE  t_num (
+#   t_pk		INT PRIMARY KEY,
+#   t_bit		BIT(64), 
+#   t_tinyint	TINYINT, 
+#   t_bool	BOOL, 
+#   t_smallint	SMALLINT,
+#   t_mediumint	MEDIUMINT,
+#   t_int		INT,
+#   t_bigint	BIGINT,
+#   t_float	FLOAT,
+#   t_double	DOUBLE,
+#   t_decimal	DECIMAL (37, 16)
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_num VALUE (
+#   1,
+#   b'1010101010101010101010101010101010101010101010101010101010101010',
+#   125,
+#   1,
+#   32765,
+#   8388606,
+#   2147483647,
+#   9223372036854775807,
+#   1e+20,
+#   1e+150,
+#   '331.0000000000'
+# );
+# 
+# CREATE TABLE  t_datetime (
+#   t_pk		INT PRIMARY KEY,
+#   t_date	DATE,
+#   t_datetime	DATETIME,
+#   t_timestamp	TIMESTAMP,
+#   t_time	TIME,
+#   t_year	YEAR
+# ) ENGINE=NDBCLUSTER;
+#   
+# INSERT INTO t_datetime VALUE (
+#   1,
+#   '1998-01-01',
+#   '2006-08-10 10:11:12',
+#   20021029165106,
+#   '19:38:34',
+#   2155
+# );
+# 
+# CREATE TABLE  t_string_1 (
+#   t_pk		INT PRIMARY KEY,
+#   t_char	CHAR(255),
+#   t_varchar	VARCHAR(655),
+#   t_binary	BINARY(255),
+#   t_varbinary	VARBINARY(6553)
+# ) ENGINE=NDBCLUSTER;
+# 
+# CREATE TABLE  t_string_2 (
+#   t_pk		INT PRIMARY KEY,
+#   t_tinyblob	TINYBLOB,
+#   t_tinytext	TINYTEXT,
+#   t_blob	BLOB,
+#   t_text	TEXT,
+#   t_mediumblob	MEDIUMBLOB,
+#   t_mediumtext	MEDIUMTEXT,
+#   t_longblob	LONGBLOB,
+#   t_longtext	LONGTEXT,
+#   t_enum	ENUM('001001','001004','001010','001018','001019','001020','001021','001027','001028','001029','001030','001031','001100','002003','002004','002005','002007')DEFAULT '001001' NOT NULL,
+#   t_set		SET('a','B')
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_string_1 VALUE (
+#   1,
+#   'abcdefghijklmn',
+#   'abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn',
+#   0x612020,
+#   0x4100
+# );
+# 
+# INSERT INTO t_string_2 VALUE (
+#   1,
+#   'abcdefghijklmnabcdefghijklmn',
+#   'abcdefghijklmnabcdefghijklmn',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   'a123456789b123456789c123456789d123456789e123456789f123456789g123456789',
+#   '001001',
+#   'a'
+# ); 
+#  
+# CREATE TABLE t_gis (
+#   t_pk			INT PRIMARY KEY,
+#   t_point		POINT,
+#   t_linestring		LINESTRING,
+#   t_polygon		POLYGON,
+#   t_multipoint		MULTIPOINT,
+#   t_multilinestring	MULTILINESTRING,
+#   t_multipolygon	MULTIPOLYGON,
+#   t_geometrycollection	GEOMETRYCOLLECTION,
+#   t_geometry		GEOMETRY
+# ) ENGINE=NDBCLUSTER;
+# 
+# INSERT INTO t_gis VALUE (
+#   1,
+#   PointFromText('POINT(10 10)'),
+#   LineStringFromText('LINESTRING(10 10,20 10,20 20,10 20,10 10)'),
+#   PolyFromText('POLYGON((0 0,50 0,50 50,0 50,0 0), (10 10,20 10,20 20,10 20,10 10))'),
+#   MPointFromText('MULTIPOINT(1 1,11 11,11 21,21 21)'),
+#   MLineFromText('MULTILINESTRING((10 48,10 21,10 0))'),
+#   MPolyFromText('MULTIPOLYGON(((28 26,28 0,84 0,84 42,28 26),(52 18,66 23,73 9,48 6,52 18)),((59 18,67 18,67 13,59 13,59 18)))'),
+#   GeomCollFromText('GEOMETRYCOLLECTION(POINT(0 0), LINESTRING(0 0,10 10))'),
+#   MPolyFromText('MULTIPOLYGON(((28 26,28 0,84 0,84 42,28 26),(52 18,66 23,73 9,48 6,52 18)),((59 18,67 18,67 13,59 13,59 18)))')
+# );
+# 
+# INSERT INTO t_gis VALUE (
+#   2,
+#   PointFromWKB(AsWKB(PointFromText('POINT(10 20)'))),
+#   LineStringFromWKB(LineString(Point(10, 10), Point(40, 10))),
+#   PolyFromWKB(Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0)))),
+#   MPointFromWKB(MultiPoint(Point(3, 6), Point(4, 10))),
+#   MLineFromWKB(MultiLineString(LineString(Point(1, 2), Point(3, 5)), LineString(Point(2, 5), Point(5, 8), Point(21, 7)))),
+#   MPolyFromWKB(MultiPolygon(Polygon(LineString(Point(0, 3), Point(3, 3), Point(3, 0), Point(0, 3))))),
+#   GeometryFromWKB(GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9)))),
+#   GeometryFromWKB(GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9))))
+# );
+
+
+
+#
+# Restore backup files (from little endian)
+#
+
+--disable_warnings
+USE test;
+DROP TABLE IF EXISTS t_num,t_datetime,t_string_1,t_string_2,t_gis;
+--enable_warnings
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 1 -m -r $MYSQL_TEST_DIR/std_data/ndb_backup51_data_le >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 2 -r $MYSQL_TEST_DIR/std_data/ndb_backup51_data_le >> $NDB_TOOLS_OUTPUT
+SHOW TABLES;
+SHOW CREATE TABLE t_num;
+SHOW CREATE TABLE t_datetime;
+SHOW CREATE TABLE t_string_1;
+SHOW CREATE TABLE t_string_2;
+SHOW CREATE TABLE t_gis;
+SELECT * FROM t_datetime;
+SELECT t_pk,hex(t_bit),t_tinyint,t_bool,t_smallint,t_mediumint,t_int,t_bigint,t_float,t_double,t_decimal FROM t_num;
+SELECT t_pk,t_char,t_varchar,hex(t_binary),hex(t_varbinary) FROM t_string_1;
+SELECT * FROM t_string_2;
+SELECT AsText(t_point), AsText(t_linestring),AsText(t_polygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_multipoint), AsText(t_multilinestring),AsText(t_multipolygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_geometrycollection), AsText(t_geometry) FROM t_gis ORDER by t_pk;
+
+#
+# Restore backup files (from big endian)
+#
+
+DROP TABLE t_num,t_datetime,t_string_1,t_string_2,t_gis;
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 1 -m -r $MYSQL_TEST_DIR/std_data/ndb_backup51_data_be >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_restore --no-defaults -b 1 -n 2 -r $MYSQL_TEST_DIR/std_data/ndb_backup51_data_be >> $NDB_TOOLS_OUTPUT
+SHOW TABLES;
+SHOW CREATE TABLE t_num;
+SHOW CREATE TABLE t_datetime;
+SHOW CREATE TABLE t_string_1;
+SHOW CREATE TABLE t_string_2;
+SHOW CREATE TABLE t_gis;
+SELECT * FROM t_datetime;
+SELECT t_pk,hex(t_bit),t_tinyint,t_bool,t_smallint,t_mediumint,t_int,t_bigint,t_float,t_double,t_decimal FROM t_num;
+SELECT t_pk,t_char,t_varchar,hex(t_binary),hex(t_varbinary) FROM t_string_1;
+SELECT * FROM t_string_2;
+SELECT AsText(t_point), AsText(t_linestring),AsText(t_polygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_multipoint), AsText(t_multilinestring),AsText(t_multipolygon) FROM t_gis ORDER by t_pk;
+SELECT AsText(t_geometrycollection), AsText(t_geometry) FROM t_gis ORDER by t_pk;
+
+DROP TABLE t_num,t_datetime,t_string_1,t_string_2,t_gis;

storage/ndb/tools/restore/Restore.cpp

@@ -667,6 +667,20 @@ RestoreDataIterator::getNextTuple(int  & res)
      */
     const Uint32 arraySize = sz / (attr_desc->size / 8);
     assert(arraySize <= attr_desc->arraySize);
+
+    //convert the length of blob(v1) and text(v1)
+    if(!m_hostByteOrder
+        && (attr_desc->m_column->getType() == NdbDictionary::Column::Blob
+           || attr_desc->m_column->getType() == NdbDictionary::Column::Text)
+        && attr_desc->m_column->getArrayType() == NdbDictionary::Column::ArrayTypeFixed)
+    {
+      char* p = (char*)&attr_data->u_int64_value[0];
+      Uint64 x;
+      memcpy(&x, p, sizeof(Uint64));
+      x = Twiddle64(x);
+      memcpy(p, &x, sizeof(Uint64));
+    }
+
     if(!Twiddle(attr_desc, attr_data, attr_desc->arraySize))
       {
 	res = -1;