WL#3228 (NDB) : RBR using different table defs on slave/master

This patch corrects a problem found during testing on Solaris. The code
changes how length values are retrieved on big endian machines. The
patch allows the rpl_extraColmaster tests to run on these machines.


mysql-test/suite/rpl/r/rpl_row_create_table.result:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  New result file with changes from merge of 5.1 main.
mysql-test/suite/rpl/t/disabled.def:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  Disable the rpl_rwo_extraColmaster_ndb test (WL#3915) because the code
  fails on Big Endian machines. See BUG#29549 for more details.
sql/field.cc:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  This patch corrects a problem found during testing on Solaris. The code
  changes how the store_length method processes requests for values on
  big endian machines.
sql/field.h:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  This patch corrects a problem found during testing on Solaris. The code
  changes how the store_length method processes requests for values on
  big endian machines. It also changes the get_packed_length() method to
  use the endian-ness of the host in getting the length + packlength.
sql/rpl_record.cc:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  This patch turns on the little endian switch (db_low_byte_first) in 
  order to ensure the values are unpack correctly from binlog as they
  are stored in little endian format in binlog.
sql/rpl_utility.cc:
  WL#3228 (NDB) : RBR using different table defs on slave/master
  
  This patch corrects a problem found during testing on Solaris. The code
  changes how the calculated field size method processes requests for 
  values on big endian machines.

mysql-test/suite/rpl/r/rpl_row_create_table.result

@@ -400,12 +400,12 @@ SHOW BINLOG EVENTS FROM 637;
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
 #	637	Query	#	717	use `test`; TRUNCATE TABLE t2
 #	717	Xid	#	744	COMMIT /* XID */
-#	738	Query	#	806	use `test`; BEGIN
-#	806	Table_map	#	845	table_id: # (test.t2)
-#	845	Write_rows	#	889	table_id: # flags: STMT_END_F
-#	889	Table_map	#	928	table_id: # (test.t2)
-#	928	Write_rows	#	967	table_id: # flags: STMT_END_F
-#	967	Query	#	1038	use `test`; ROLLBACK
+#	744	Query	#	812	use `test`; BEGIN
+#	812	Table_map	#	853	table_id: # (test.t2)
+#	853	Write_rows	#	897	table_id: # flags: STMT_END_F
+#	897	Table_map	#	938	table_id: # (test.t2)
+#	938	Write_rows	#	977	table_id: # flags: STMT_END_F
+#	977	Query	#	1048	use `test`; ROLLBACK
 SELECT * FROM t2 ORDER BY a;
 a
 DROP TABLE t1,t2;

mysql-test/suite/rpl/t/disabled.def

@@ -16,3 +16,4 @@ rpl_innodb_mixed_dml     : Bug #29363 rpl.rpl_innodb_mixed_* test failures
 rpl_invoked_features     : BUG#29020 2007-06-21 Lars Non-deterministic test case
 rpl_auto_increment_11932   : Bug#29809 2007-07-16 ingo Slave SQL errors in warnings file
 rpl_stm_extraColmaster_ndb : WL#3915 : Statement-based replication not supported in ndb. Enable test when supported.
+rpl_row_extraColmaster_ndb : BUG#29549 : Replication of BLOBs fail for NDB

sql/field.cc

@@ -7181,7 +7181,10 @@ Field_blob::Field_blob(uchar *ptr_arg, uchar *null_ptr_arg, uchar null_bit_arg,
 }
 
 
-void Field_blob::store_length(uchar *i_ptr, uint i_packlength, uint32 i_number)
+void Field_blob::store_length(uchar *i_ptr, 
+                              uint i_packlength, 
+                              uint32 i_number, 
+                              bool low_byte_first)
 {
   switch (i_packlength) {
   case 1:
@@ -7189,7 +7192,7 @@ void Field_blob::store_length(uchar *i_ptr, uint i_packlength, uint32 i_number)
     break;
   case 2:
 #ifdef WORDS_BIGENDIAN
-    if (table->s->db_low_byte_first)
+    if (low_byte_first)
     {
       int2store(i_ptr,(unsigned short) i_number);
     }
@@ -7202,7 +7205,7 @@ void Field_blob::store_length(uchar *i_ptr, uint i_packlength, uint32 i_number)
     break;
   case 4:
 #ifdef WORDS_BIGENDIAN
-    if (table->s->db_low_byte_first)
+    if (low_byte_first)
     {
       int4store(i_ptr,i_number);
     }
@@ -7213,7 +7216,7 @@ void Field_blob::store_length(uchar *i_ptr, uint i_packlength, uint32 i_number)
 }
 
 
-uint32 Field_blob::get_length(const uchar *pos)
+uint32 Field_blob::get_length(const uchar *pos, bool low_byte_first)
 {
   switch (packlength) {
   case 1:
@@ -7222,7 +7225,7 @@ uint32 Field_blob::get_length(const uchar *pos)
     {
       uint16 tmp;
 #ifdef WORDS_BIGENDIAN
-      if (table->s->db_low_byte_first)
+      if (low_byte_first)
 	tmp=sint2korr(pos);
       else
 #endif
@@ -7235,7 +7238,7 @@ uint32 Field_blob::get_length(const uchar *pos)
     {
       uint32 tmp;
 #ifdef WORDS_BIGENDIAN
-      if (table->s->db_low_byte_first)
+      if (low_byte_first)
 	tmp=uint4korr(pos);
       else
 #endif
@@ -7740,7 +7743,7 @@ uchar *Field_blob::pack_key(uchar *to, const uchar *from, uint max_length)
 {
   uchar *save= ptr;
   ptr= (uchar*) from;
-  uint32 length=get_length();			// Length of from string
+  uint32 length=get_length();        // Length of from string
   uint local_char_length= ((field_charset->mbmaxlen > 1) ?
                            max_length/field_charset->mbmaxlen : max_length);
   if (length)

sql/field.h

@@ -1344,7 +1344,11 @@ public:
 #ifndef WORDS_BIGENDIAN
   static
 #endif
-  void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number);
+  void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number, bool low_byte_first);
+  void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number)
+  {
+    store_length(i_ptr, i_packlength, i_number, table->s->db_low_byte_first);
+  }
   inline void store_length(uint32 number)
   {
     store_length(ptr, packlength, number);
@@ -1358,12 +1362,14 @@ public:
 
      @retval The length in the row plus the size of the data.
   */
-  uint32 get_packed_size(const uchar *ptr)
-    {return packlength + get_length((const uchar *)ptr);}
-
-  inline uint32 get_length(uint row_offset=0)
-  { return get_length(ptr+row_offset); }
-  uint32 get_length(const uchar *ptr);
+  uint32 get_packed_size(const uchar *ptr_arg, bool low_byte_first)
+    {return packlength + get_length(ptr_arg, low_byte_first);}
+
+  inline uint32 get_length(uint row_offset= 0)
+  { return get_length(ptr+row_offset, table->s->db_low_byte_first); }
+  uint32 get_length(const uchar *ptr, bool low_byte_first);
+  uint32 get_length(const uchar *ptr_arg)
+  { return get_length(ptr_arg, table->s->db_low_byte_first); }
   void put_length(uchar *pos, uint32 length);
   inline void get_ptr(uchar **str)
     {
@@ -1395,7 +1401,8 @@ public:
   {
     uchar *tmp;
     get_ptr(&tmp);
-    if (value.copy((char*) tmp, get_length(),charset()))
+    uint32 len= get_length(0, table->s->db_low_byte_first);
+    if (value.copy((char*) tmp, len, charset()))
     {
       Field_blob::reset();
       return 1;

sql/rpl_record.cc

@@ -92,8 +92,30 @@ pack_row(TABLE *table, MY_BITMAP const* cols,
 
         /*
           We only store the data of the field if it is non-null
-         */
+
+          For big-endian machines, we have to make sure that the
+          length is stored in little-endian format, since this is the
+          format used for the binlog.
+
+          We do this by setting the db_low_byte_first, which is used
+          inside some store_length() to decide what order to write the
+          bytes in.
+
+          In reality, db_log_byte_first is only set for legacy table
+          type Isam, but in the event of a bug, we need to guarantee
+          the endianess when writing to the binlog.
+
+          This is currently broken for NDB due to BUG#29549, so we
+          will fix it when NDB has fixed their way of handling BLOBs.
+        */
+#if 0
+        bool save= table->s->db_low_byte_first;
+        table->s->db_low_byte_first= TRUE;
+#endif
         pack_ptr= field->pack(pack_ptr, field->ptr + offset);
+#if 0
+        table->s->db_low_byte_first= save;
+#endif
       }
 
       null_mask <<= 1;
@@ -229,10 +251,17 @@ unpack_row(RELAY_LOG_INFO const *rli,
           Use the master's size information if available else call
           normal unpack operation.
          */
+#if 0
+         bool save= table->s->db_low_byte_first;
+         table->s->db_low_byte_first= TRUE;
+#endif
         if (tabledef && tabledef->field_metadata(i))
           pack_ptr= f->unpack(f->ptr, pack_ptr, tabledef->field_metadata(i));
         else
           pack_ptr= f->unpack(f->ptr, pack_ptr);
+#if 0
+        table->s->db_low_byte_first= save;
+#endif
       }
 
       bitmap_set_bit(rw_set, f->field_index);

sql/rpl_utility.cc

@@ -114,8 +114,43 @@ uint32 table_def::calc_field_size(uint col, uchar *master_data)
   case MYSQL_TYPE_BLOB:
   case MYSQL_TYPE_GEOMETRY:
   {
+#if 1
+    /*
+      BUG#29549: 
+      This is currently broken for NDB, which is using big-endian
+      order when packing length of BLOB. Once they have decided how to
+      fix the issue, we can enable the code below to make sure to
+      always read the length in little-endian order.
+    */
     Field_blob fb(m_field_metadata[col]);
-    length= fb.get_packed_size(master_data);
+    length= fb.get_packed_size(master_data, TRUE);
+#else
+    /*
+      Compute the length of the data. We cannot use get_length() here
+      since it is dependent on the specific table (and also checks the
+      packlength using the internal 'table' pointer) and replication
+      is using a fixed format for storing data in the binlog.
+    */
+    switch (m_field_metadata[col]) {
+    case 1:
+      length= *master_data;
+      break;
+    case 2:
+      length= sint2korr(master_data);
+      break;
+    case 3:
+      length= uint3korr(master_data);
+      break;
+    case 4:
+      length= uint4korr(master_data);
+      break;
+    default:
+      DBUG_ASSERT(0);		// Should not come here
+      break;
+    }
+
+    length+= m_field_metadata[col];
+#endif
     break;
   }
   default: