BUG#52868: Wrong handling of NULL value during update, replication out

           of sync

In RBR, sometimes the table->s->last_null_bit_pos can be zero. This
has impact at the slave when it compares records fetched from the
storage engine against records in the binary log event. If
last_null_bit_pos is zero the slave, while comparing in
log_event.cc:record_compare function, would set all bits in the last
null_byte to 1 (assumed all 8 were unused) . Thence it would loose the
ability to distinguish records that were similar in contents except
for the fact that some field was null in one record, but not in the
other. Ultimately this would cause wrong matches, and in the specific
case depicted in the bug report the same record would be updated
twice, resulting in a lost update.

Additionally, in the record_compare function the slave was setting the
X bit unconditionally. There are cases that the X bit does not exist
in the record header. This could also lead to wrong matches between
records.

We fix both by conditionally resetting the bits: (i) unused null_bits
are set if last_null_bit_pos > 0; (ii) X bit is set if
HA_OPTION_PACK_RECORD is in use.

mysql-test/extra/rpl_tests/rpl_record_compare.test

+
+#
+# BUG#52868: Wrong handling of NULL value during update, replication out of sync
+#
+-- echo ## case #1 - last_null_bit_pos==0 in record_compare without X bit
+
+-- source include/master-slave-reset.inc
+-- connection master
+
+-- eval CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 varchar(1) DEFAULT '', c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0, c8 bigint(20) DEFAULT 0) ENGINE=$engine DEFAULT CHARSET=latin1
+
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+-- disable_warnings
+UPDATE t1 SET c5 = 'a';
+-- enable_warnings
+-- sync_slave_with_master
+
+-- let $diff_table_1= master:test.t1
+-- let $diff_table_2= slave:test.t1
+-- source include/diff_tables.inc
+
+--connection master
+DROP TABLE t1;
+-- sync_slave_with_master
+
+-- echo ## case #1.1 - last_null_bit_pos==0 in record_compare with X bit
+-- echo ##             (1 column less and no varchar)
+-- source include/master-slave-reset.inc
+-- connection master
+
+-- eval CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 bigint(20) DEFAULT 0, c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0) ENGINE=$engine DEFAULT CHARSET=latin1
+
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+-- disable_warnings
+UPDATE t1 SET c5 = 'a';
+-- enable_warnings
+-- sync_slave_with_master
+
+-- let $diff_table_1= master:test.t1
+-- let $diff_table_2= slave:test.t1
+-- source include/diff_tables.inc
+
+--connection master
+DROP TABLE t1;
+-- sync_slave_with_master
+
+-- echo ## case #2 - X bit is wrongly set.
+
+-- source include/master-slave-reset.inc
+-- connection master
+
+-- eval CREATE TABLE t1 (c1 int, c2 varchar(1) default '') ENGINE=$engine DEFAULT CHARSET= latin1
+INSERT INTO t1(c1) VALUES (10);
+INSERT INTO t1(c1) VALUES (NULL);
+UPDATE t1 SET c1= 0;
+-- sync_slave_with_master
+
+-- let $diff_table_1= master:test.t1
+-- let $diff_table_2= slave:test.t1
+-- source include/diff_tables.inc
+
+-- connection master
+DROP TABLE t1; 
+-- sync_slave_with_master
+
+

mysql-test/suite/rpl/r/rpl_row_rec_comp_innodb.result

+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+## case #1 - last_null_bit_pos==0 in record_compare without X bit
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 varchar(1) DEFAULT '', c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0, c8 bigint(20) DEFAULT 0) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+UPDATE t1 SET c5 = 'a';
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;
+## case #1.1 - last_null_bit_pos==0 in record_compare with X bit
+##             (1 column less and no varchar)
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 bigint(20) DEFAULT 0, c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+UPDATE t1 SET c5 = 'a';
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;
+## case #2 - X bit is wrongly set.
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 int, c2 varchar(1) default '') ENGINE=InnoDB DEFAULT CHARSET= latin1;
+INSERT INTO t1(c1) VALUES (10);
+INSERT INTO t1(c1) VALUES (NULL);
+UPDATE t1 SET c1= 0;
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;

mysql-test/suite/rpl/r/rpl_row_rec_comp_myisam.result

+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+## case #1 - last_null_bit_pos==0 in record_compare without X bit
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 varchar(1) DEFAULT '', c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0, c8 bigint(20) DEFAULT 0) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+UPDATE t1 SET c5 = 'a';
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;
+## case #1.1 - last_null_bit_pos==0 in record_compare with X bit
+##             (1 column less and no varchar)
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bigint(20) DEFAULT 0, c3 bigint(20) DEFAULT 0, c4 bigint(20) DEFAULT 0, c5 bigint(20) DEFAULT 0, c6 bigint(20) DEFAULT 0, c7 bigint(20) DEFAULT 0) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 ( c5, c6 ) VALUES ( 1   , 35 );
+INSERT INTO t1 ( c5, c6 ) VALUES ( NULL, 35 );
+UPDATE t1 SET c5 = 'a';
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;
+## case #2 - X bit is wrongly set.
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 int, c2 varchar(1) default '') ENGINE=MyISAM DEFAULT CHARSET= latin1;
+INSERT INTO t1(c1) VALUES (10);
+INSERT INTO t1(c1) VALUES (NULL);
+UPDATE t1 SET c1= 0;
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;
+## coverage purposes - Field_bits 
+##                     1 X bit + 2 Null bits + 5 bits => last_null_bit_pos==0
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bit(5)) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1(c1,c2) VALUES (10, b'1');
+INSERT INTO t1(c1,c2) VALUES (NULL, b'1');
+UPDATE t1 SET c1= 0;
+Comparing tables master:test.t1 and slave:test.t1
+DROP TABLE t1;

mysql-test/suite/rpl/t/rpl_row_rec_comp_innodb.test

+-- source include/have_binlog_format_row.inc
+-- source include/master-slave.inc
+-- source include/have_innodb.inc
+
+#
+# BUG#52868 Wrong handling of NULL value during update, replication out of sync
+#
+
+-- let $engine= InnoDB
+-- source extra/rpl_tests/rpl_record_compare.test

mysql-test/suite/rpl/t/rpl_row_rec_comp_myisam.test

+-- source include/have_binlog_format_row.inc
+-- source include/master-slave.inc
+
+#
+# BUG#52868 Wrong handling of NULL value during update, replication out of sync
+#
+
+-- let $engine= MyISAM
+-- source extra/rpl_tests/rpl_record_compare.test
+
+-- echo ## coverage purposes - Field_bits 
+-- echo ##                     1 X bit + 2 Null bits + 5 bits => last_null_bit_pos==0
+## Added here because AFAIK it's only MyISAM and NDB that use Field_bits
+
+-- source include/master-slave-reset.inc
+-- connection master
+
+-- eval CREATE TABLE t1 (c1 bigint(20) DEFAULT 0, c2 bit(5)) ENGINE=$engine DEFAULT CHARSET=latin1
+
+INSERT INTO t1(c1,c2) VALUES (10, b'1');
+INSERT INTO t1(c1,c2) VALUES (NULL, b'1');
+UPDATE t1 SET c1= 0;
+-- sync_slave_with_master
+
+-- let $diff_table_1= master:test.t1
+-- let $diff_table_2= slave:test.t1
+-- source include/diff_tables.inc
+
+-- connection master
+DROP TABLE t1; 
+-- sync_slave_with_master

sql/log_event.cc

@@ -8758,11 +8758,28 @@ static bool record_compare(TABLE *table)
   {
     for (int i = 0 ; i < 2 ; ++i)
     {
-      saved_x[i]= table->record[i][0];
-      saved_filler[i]= table->record[i][table->s->null_bytes - 1];
-      table->record[i][0]|= 1U;
-      table->record[i][table->s->null_bytes - 1]|=
-        256U - (1U << table->s->last_null_bit_pos);
+      /* 
+        If we have an X bit then we need to take care of it.
+      */
+      if (!(table->s->db_options_in_use & HA_OPTION_PACK_RECORD))
+      {
+        saved_x[i]= table->record[i][0];
+        table->record[i][0]|= 1U;
+      }
+
+      /*
+         If (last_null_bit_pos == 0 && null_bytes > 1), then:
+
+         X bit (if any) + N nullable fields + M Field_bit fields = 8 bits 
+
+         Ie, the entire byte is used.
+      */
+      if (table->s->last_null_bit_pos > 0)
+      {
+        saved_filler[i]= table->record[i][table->s->null_bytes - 1];
+        table->record[i][table->s->null_bytes - 1]|=
+          256U - (1U << table->s->last_null_bit_pos);
+      }
     }
   }
 
@@ -8802,8 +8819,11 @@ record_compare_exit:
   {
     for (int i = 0 ; i < 2 ; ++i)
     {
-      table->record[i][0]= saved_x[i];
-      table->record[i][table->s->null_bytes - 1]= saved_filler[i];
+      if (!(table->s->db_options_in_use & HA_OPTION_PACK_RECORD))
+        table->record[i][0]= saved_x[i];
+
+      if (table->s->last_null_bit_pos)
+        table->record[i][table->s->null_bytes - 1]= saved_filler[i];
     }
   }
 

sql/log_event_old.cc

@@ -342,12 +342,29 @@ static bool record_compare(TABLE *table)
   if (table->s->null_bytes > 0)
   {
     for (int i = 0 ; i < 2 ; ++i)
-    {
-      saved_x[i]= table->record[i][0];
-      saved_filler[i]= table->record[i][table->s->null_bytes - 1];
-      table->record[i][0]|= 1U;
-      table->record[i][table->s->null_bytes - 1]|=
-        256U - (1U << table->s->last_null_bit_pos);
+    { 
+      /* 
+        If we have an X bit then we need to take care of it.
+      */
+      if (!(table->s->db_options_in_use & HA_OPTION_PACK_RECORD))
+      {
+        saved_x[i]= table->record[i][0];
+        table->record[i][0]|= 1U;
+      }
+      
+      /*
+         If (last_null_bit_pos == 0 && null_bytes > 1), then:
+
+         X bit (if any) + N nullable fields + M Field_bit fields = 8 bits 
+
+         Ie, the entire byte is used.
+      */
+      if (table->s->last_null_bit_pos > 0)
+      {
+        saved_filler[i]= table->record[i][table->s->null_bytes - 1];
+        table->record[i][table->s->null_bytes - 1]|=
+          256U - (1U << table->s->last_null_bit_pos);
+      }
     }
   }
 
@@ -387,8 +404,11 @@ record_compare_exit:
   {
     for (int i = 0 ; i < 2 ; ++i)
     {
-      table->record[i][0]= saved_x[i];
-      table->record[i][table->s->null_bytes - 1]= saved_filler[i];
+      if (!(table->s->db_options_in_use & HA_OPTION_PACK_RECORD))
+        table->record[i][0]= saved_x[i];
+
+      if (table->s->last_null_bit_pos > 0)
+        table->record[i][table->s->null_bytes - 1]= saved_filler[i];
     }
   }