MDEV-9242: Innodb reports Assertion failure in file buf0dblwr.cc line 579

Analysis: When pages in doublewrite buffer are analyzed compressed
pages do not have correct checksum.

Fix: Decompress page before checksum is compared. If decompression
fails we still check checksum and corrupted pages are found.
If decompression succeeds, page now contains the original
checksum.

storage/innobase/buf/buf0dblwr.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -37,6 +37,7 @@ Created 2011/12/19
 #include "page0zip.h"
 #include "trx0sys.h"
 #include "fil0crypt.h"
+#include "fil0pagecompress.h"
 
 #ifndef UNIV_HOTBACKUP
 
@@ -500,6 +501,7 @@ buf_dblwr_process()
 
 	for (std::list<byte*>::iterator i = recv_dblwr.pages.begin();
 	     i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) {
+		bool is_compressed = false;
 
 		page = *i;
 		page_no  = mach_read_from_4(page + FIL_PAGE_OFFSET);
@@ -533,6 +535,16 @@ buf_dblwr_process()
 				NULL,
 				0);
 
+			/* Is page compressed ? */
+			is_compressed = fil_page_is_compressed_encrypted(read_buf) |
+				fil_page_is_compressed(read_buf);
+
+			/* If page was compressed, decompress it before we
+			check checksum. */
+			if (is_compressed) {
+				fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true);
+			}
+
 			if (fil_space_verify_crypt_checksum(read_buf, zip_size)) {
 				/* page is encrypted and checksum is OK */
 			} else if (buf_page_is_corrupted(true, read_buf, zip_size)) {
@@ -546,6 +558,16 @@ buf_dblwr_process()
 					" the doublewrite buffer.\n",
 					(ulong) space_id, (ulong) page_no);
 
+				/* Is page compressed ? */
+				is_compressed = fil_page_is_compressed_encrypted(page) |
+					fil_page_is_compressed(page);
+
+				/* If page was compressed, decompress it before we
+				check checksum. */
+				if (is_compressed) {
+					fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, NULL, true);
+				}
+
 				if (fil_space_verify_crypt_checksum(page, zip_size)) {
 					/* the doublewrite buffer page is encrypted and OK */
 				} else if (buf_page_is_corrupted(true,

storage/innobase/fil/fil0pagecompress.cc

 /*****************************************************************************
 
-Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2013, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -447,8 +447,11 @@ fil_decompress_page(
 	byte*	buf,		/*!< out: buffer from which to read; in aio
 				this must be appropriately aligned */
 	ulong	len,		/*!< in: length of output buffer.*/
-	ulint*	write_size)	/*!< in/out: Actual payload size of
+	ulint*	write_size,	/*!< in/out: Actual payload size of
 				the compressed data. */
+	bool	return_error)	/*!< in: true if only an error should
+				be produced when decompression fails.
+				By default this parameter is false. */
 {
 	int err = 0;
 	ulint actual_size = 0;
@@ -493,6 +496,9 @@ fil_decompress_page(
 			mach_read_from_2(buf+FIL_PAGE_TYPE), len);
 
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 	}
 
@@ -512,6 +518,9 @@ fil_decompress_page(
 			" actual size %lu compression %s.",
 			actual_size, fil_get_compression_alg_name(compression_alg));
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 	}
 
@@ -543,6 +552,9 @@ fil_decompress_page(
 
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -559,6 +571,9 @@ fil_decompress_page(
 				err, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -577,6 +592,9 @@ fil_decompress_page(
 				olen, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -610,6 +628,9 @@ fil_decompress_page(
 				dst_pos, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 
@@ -636,6 +657,9 @@ fil_decompress_page(
 				dst_pos, actual_size, len, err);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -661,6 +685,9 @@ fil_decompress_page(
 				olen, actual_size, len, (int)cstatus);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -674,6 +701,9 @@ fil_decompress_page(
 			,fil_get_compression_alg_name(compression_alg));
 
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 		break;
 	}
@@ -684,6 +714,7 @@ fil_decompress_page(
 	really any other options. */
 	memcpy(buf, in_buf, len);
 
+error_return:
 	// Need to free temporal buffer if no buffer was given
 	if (page_buf == NULL) {
 		ut_free(in_buf);

storage/innobase/include/fil0pagecompress.h

 /*****************************************************************************
 
-Copyright (C) 2013, 2015 MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -117,8 +117,12 @@ fil_decompress_page(
 	byte*	buf,		/*!< out: buffer from which to read; in aio
 				this must be appropriately aligned */
 	ulong	len,		/*!< in: length of output buffer.*/
-	ulint*	write_size);	/*!< in/out: Actual payload size of
+	ulint*	write_size,	/*!< in/out: Actual payload size of
 				the compressed data. */
+	bool	return_error=false);
+				/*!< in: true if only an error should
+				be produced when decompression fails.
+				By default this parameter is false. */
 
 /****************************************************************//**
 Get space id from fil node

storage/xtradb/buf/buf0dblwr.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -37,6 +37,7 @@ Created 2011/12/19
 #include "page0zip.h"
 #include "trx0sys.h"
 #include "fil0crypt.h"
+#include "fil0pagecompress.h"
 
 #ifndef UNIV_HOTBACKUP
 
@@ -500,6 +501,7 @@ buf_dblwr_process()
 
 	for (std::list<byte*>::iterator i = recv_dblwr.pages.begin();
 	     i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) {
+		bool is_compressed = false;
 
 		page = *i;
 		page_no  = mach_read_from_4(page + FIL_PAGE_OFFSET);
@@ -533,6 +535,16 @@ buf_dblwr_process()
 				NULL,
 				0);
 
+			/* Is page compressed ? */
+			is_compressed = fil_page_is_compressed_encrypted(read_buf) |
+				fil_page_is_compressed(read_buf);
+
+			/* If page was compressed, decompress it before we
+			check checksum. */
+			if (is_compressed) {
+				fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true);
+			}
+
 			if (fil_space_verify_crypt_checksum(read_buf, zip_size)) {
 				/* page is encrypted and checksum is OK */
 			} else if (buf_page_is_corrupted(true, read_buf, zip_size)) {
@@ -546,6 +558,16 @@ buf_dblwr_process()
 					" the doublewrite buffer.\n",
 					(ulong) space_id, (ulong) page_no);
 
+				/* Is page compressed ? */
+				is_compressed = fil_page_is_compressed_encrypted(page) |
+					fil_page_is_compressed(page);
+
+				/* If page was compressed, decompress it before we
+				check checksum. */
+				if (is_compressed) {
+					fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, NULL, true);
+				}
+
 				if (fil_space_verify_crypt_checksum(page, zip_size)) {
 					/* the doublewrite buffer page is encrypted and OK */
 				} else if (buf_page_is_corrupted(true,

storage/xtradb/fil/fil0pagecompress.cc

 /*****************************************************************************
 
-Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2013, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -446,8 +446,11 @@ fil_decompress_page(
 	byte*	buf,		/*!< out: buffer from which to read; in aio
 				this must be appropriately aligned */
 	ulong	len,		/*!< in: length of output buffer.*/
-	ulint*	write_size)	/*!< in/out: Actual payload size of
+	ulint*	write_size,	/*!< in/out: Actual payload size of
 				the compressed data. */
+	bool	return_error)	/*!< in: true if only an error should
+				be produced when decompression fails.
+				By default this parameter is false. */
 {
 	int err = 0;
 	ulint actual_size = 0;
@@ -492,6 +495,9 @@ fil_decompress_page(
 			mach_read_from_2(buf+FIL_PAGE_TYPE), len);
 
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 	}
 
@@ -511,6 +517,9 @@ fil_decompress_page(
 			" actual size %lu compression %s.",
 			actual_size, fil_get_compression_alg_name(compression_alg));
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 	}
 
@@ -542,6 +551,9 @@ fil_decompress_page(
 
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -558,6 +570,9 @@ fil_decompress_page(
 				err, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -576,6 +591,9 @@ fil_decompress_page(
 				olen, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -609,6 +627,9 @@ fil_decompress_page(
 				dst_pos, actual_size, len);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 
@@ -635,6 +656,9 @@ fil_decompress_page(
 				dst_pos, actual_size, len, err);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -660,6 +684,9 @@ fil_decompress_page(
 				olen, actual_size, len, (int)cstatus);
 			fflush(stderr);
 
+			if (return_error) {
+				goto error_return;
+			}
 			ut_error;
 		}
 		break;
@@ -673,6 +700,9 @@ fil_decompress_page(
 			,fil_get_compression_alg_name(compression_alg));
 
 		fflush(stderr);
+		if (return_error) {
+			goto error_return;
+		}
 		ut_error;
 		break;
 	}
@@ -683,6 +713,7 @@ fil_decompress_page(
 	really any other options. */
 	memcpy(buf, in_buf, len);
 
+error_return:
 	// Need to free temporal buffer if no buffer was given
 	if (page_buf == NULL) {
 		ut_free(in_buf);

storage/xtradb/include/fil0pagecompress.h

 /*****************************************************************************
 
-Copyright (C) 2013, 2015 MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -117,8 +117,12 @@ fil_decompress_page(
 	byte*	buf,		/*!< out: buffer from which to read; in aio
 				this must be appropriately aligned */
 	ulong	len,		/*!< in: length of output buffer.*/
-	ulint*	write_size);	/*!< in/out: Actual payload size of
+	ulint*	write_size,	/*!< in/out: Actual payload size of
 				the compressed data. */
+	bool	return_error=false);
+				/*!< in: true if only an error should
+				be produced when decompression fails.
+				By default this parameter is false. */
 
 /****************************************************************//**
 Get space id from fil node