Fix BUG#14747: "Race condition can cause btr_search_drop_page_hash_index()

 to crash".
 Changes from snapshot innodb-5.0-ss52.
 Note that buf_block_t::index should be protected by btr_search_latch
 or an s-latch or x-latch on the index page.
 btr_search_drop_page_hash_index(): Read block->index while holding
 btr_search_latch and use the cached value in the loop. Remove some
 redundant assertions.
 Also fix 13778. When FOREIGN_KEY_CHECKS=0 we still need to check that
 datatypes between foreign key references are compatible.
 Also added test cases to 9802.


innobase/btr/btr0sea.c:
  Changes from innodb-5.0-ss52
innobase/dict/dict0dict.c:
  Changes from innodb-5.0-ss52
innobase/dict/dict0load.c:
  Changes from innodb-5.0-ss52
innobase/include/buf0buf.h:
  Changes from innodb-5.0-ss52
innobase/include/dict0dict.h:
  Changes from innodb-5.0-ss52
innobase/include/dict0load.h:
  Changes from innodb-5.0-ss52
innobase/include/rem0cmp.h:
  Changes from innodb-5.0-ss52
innobase/rem/rem0cmp.c:
  Changes from innodb-5.0-ss52
innobase/row/row0mysql.c:
  Changes from innodb-5.0-ss52
mysql-test/r/innodb.result:
  Changes from innodb-5.0-ss52
mysql-test/t/innodb.test:
  Changes from innodb-5.0-ss52
sql/ha_innodb.cc:
  Changes from innodb-5.0-ss52
sql/ha_innodb.h:
  Changes from innodb-5.0-ss52

innobase/btr/btr0sea.c

@@ -904,6 +904,7 @@ btr_search_drop_page_hash_index(
 	ulint*		folds;
 	ulint		i;
 	mem_heap_t*	heap;
+	dict_index_t*	index;
 	ulint*		offsets;
 
 #ifdef UNIV_SYNC_DEBUG
@@ -932,11 +933,16 @@ btr_search_drop_page_hash_index(
 
 	n_fields = block->curr_n_fields;
 	n_bytes = block->curr_n_bytes;
+	index = block->index;
 
-	ut_a(n_fields + n_bytes > 0);
+	/* NOTE: The fields of block must not be accessed after
+	releasing btr_search_latch, as the index page might only
+	be s-latched! */
 
 	rw_lock_s_unlock(&btr_search_latch);
 	
+	ut_a(n_fields + n_bytes > 0);
+
 	n_recs = page_get_n_recs(page);
 
 	/* Calculate and cache fold values into an array for fast deletion
@@ -949,14 +955,6 @@ btr_search_drop_page_hash_index(
 	rec = page_get_infimum_rec(page);
 	rec = page_rec_get_next(rec);
 
-	if (!page_rec_is_supremum(rec)) {
-		ut_a(n_fields <= rec_get_n_fields(rec, block->index));
-
-		if (n_bytes > 0) {
-			ut_a(n_fields < rec_get_n_fields(rec, block->index));
-		}
-	}
-
 	tree_id = btr_page_get_index_id(page);
 	
 	prev_fold = 0;
@@ -964,18 +962,12 @@ btr_search_drop_page_hash_index(
 	heap = NULL;
 	offsets = NULL;
 
-	if (block->index == NULL) {
-
-		mem_analyze_corruption((byte*)block);
-
-		ut_a(block->index != NULL);
-	}
-
 	while (!page_rec_is_supremum(rec)) {
 		/* FIXME: in a mixed tree, not all records may have enough
 		ordering fields: */
-		offsets = rec_get_offsets(rec, block->index,
-				offsets, n_fields + (n_bytes > 0), &heap);
+		offsets = rec_get_offsets(rec, index, offsets,
+					n_fields + (n_bytes > 0), &heap);
+		ut_a(rec_offs_n_fields(offsets) == n_fields + (n_bytes > 0));
 		fold = rec_fold(rec, offsets, n_fields, n_bytes, tree_id);
 
 		if (fold == prev_fold && prev_fold != 0) {

innobase/dict/dict0dict.c

@@ -2104,8 +2104,11 @@ dict_foreign_find_index(
 	dict_table_t*	table,	/* in: table */
 	const char**	columns,/* in: array of column names */
 	ulint		n_cols,	/* in: number of columns */
-	dict_index_t*	types_idx)/* in: NULL or an index to whose types the
-				column types must match */
+	dict_index_t*	types_idx, /* in: NULL or an index to whose types the
+				   column types must match */
+	ibool		check_charsets)	/* in: whether to check charsets.
+					only has an effect if types_idx !=
+					NULL. */
 {
 #ifndef UNIV_HOTBACKUP
 	dict_index_t*	index;
@@ -2135,7 +2138,8 @@ dict_foreign_find_index(
 
 				if (types_idx && !cmp_types_are_equal(
 				     dict_index_get_nth_type(index, i),
-				     dict_index_get_nth_type(types_idx, i))) {
+				     dict_index_get_nth_type(types_idx, i),
+				     check_charsets)) {
 
 				  	break;
 				}		
@@ -2212,7 +2216,8 @@ dict_foreign_add_to_cache(
 /*======================*/
 					/* out: DB_SUCCESS or error code */
 	dict_foreign_t*	foreign,	/* in, own: foreign key constraint */
-	ibool		check_types)	/* in: TRUE=check type compatibility */
+	ibool		check_charsets)	/* in: TRUE=check charset
+					compatibility */
 {
 	dict_table_t*	for_table;
 	dict_table_t*	ref_table;
@@ -2248,16 +2253,10 @@ dict_foreign_add_to_cache(
 	}
 
 	if (for_in_cache->referenced_table == NULL && ref_table) {
-		dict_index_t*	types_idx;
-		if (check_types) {
-			types_idx = for_in_cache->foreign_index;
-		} else {
-			types_idx = NULL;
-		}
 		index = dict_foreign_find_index(ref_table,
 			(const char**) for_in_cache->referenced_col_names,
 			for_in_cache->n_fields,
-			types_idx);
+			for_in_cache->foreign_index, check_charsets);
 
 		if (index == NULL) {
 			dict_foreign_error_report(ef, for_in_cache,
@@ -2281,16 +2280,10 @@ dict_foreign_add_to_cache(
 	}
 
 	if (for_in_cache->foreign_table == NULL && for_table) {
-		dict_index_t*	types_idx;
-		if (check_types) {
-			types_idx = for_in_cache->referenced_index;
-		} else {
-			types_idx = NULL;
-		}
 		index = dict_foreign_find_index(for_table,
 			(const char**) for_in_cache->foreign_col_names,
 			for_in_cache->n_fields,
-			types_idx);
+			for_in_cache->referenced_index, check_charsets);
 
 		if (index == NULL) {
 			dict_foreign_error_report(ef, for_in_cache,
@@ -3097,7 +3090,7 @@ col_loop1:
 	/* Try to find an index which contains the columns
 	as the first fields and in the right order */
 
-	index = dict_foreign_find_index(table, column_names, i, NULL);
+	index = dict_foreign_find_index(table, column_names, i, NULL, TRUE);
 
 	if (!index) {
 		mutex_enter(&dict_foreign_err_mutex);
@@ -3362,8 +3355,7 @@ try_find_index:
 
 	if (referenced_table) {
 		index = dict_foreign_find_index(referenced_table,
-						column_names, i,
-						foreign->foreign_index);
+			column_names, i, foreign->foreign_index, TRUE);
 		if (!index) {
 			dict_foreign_free(foreign);
 			mutex_enter(&dict_foreign_err_mutex);

innobase/dict/dict0load.c

@@ -1091,7 +1091,7 @@ dict_load_foreign(
 				/* out: DB_SUCCESS or error code */
 	const char*	id,	/* in: foreign constraint id as a
 				null-terminated string */
-	ibool		check_types)/* in: TRUE=check type compatibility */
+	ibool		check_charsets)/* in: TRUE=check charset compatibility */
 {	
 	dict_foreign_t*	foreign;
 	dict_table_t*	sys_foreign;
@@ -1204,7 +1204,7 @@ dict_load_foreign(
 	a new foreign key constraint but loading one from the data
 	dictionary. */
 
-	return(dict_foreign_add_to_cache(foreign, check_types));
+	return(dict_foreign_add_to_cache(foreign, check_charsets));
 }
 
 /***************************************************************************
@@ -1219,7 +1219,8 @@ dict_load_foreigns(
 /*===============*/
 					/* out: DB_SUCCESS or error code */
 	const char*	table_name,	/* in: table name */
-	ibool		check_types)	/* in: TRUE=check type compatibility */
+	ibool		check_charsets)	/* in: TRUE=check charset
+					compatibility */
 {
 	btr_pcur_t	pcur;
 	mem_heap_t* 	heap;
@@ -1319,7 +1320,7 @@ loop:
 
 	/* Load the foreign constraint definition to the dictionary cache */
 	
-	err = dict_load_foreign(id, check_types);
+	err = dict_load_foreign(id, check_charsets);
 
 	if (err != DB_SUCCESS) {
 		btr_pcur_close(&pcur);

innobase/include/buf0buf.h

@@ -745,8 +745,6 @@ struct buf_block_struct{
 					buffer pool which are index pages,
 					but this flag is not set because
 					we do not keep track of all pages */
-	dict_index_t*	index;		/* index for which the adaptive
-					hash index has been created */
 	/* 2. Page flushing fields */
 
 	UT_LIST_NODE_T(buf_block_t) flush_list;
@@ -833,7 +831,7 @@ struct buf_block_struct{
 					records with the same prefix should be
 					indexed in the hash index */
 					
-	/* The following 4 fields are protected by btr_search_latch: */
+	/* The following 6 fields are protected by btr_search_latch: */
 
 	ibool		is_hashed;	/* TRUE if hash index has already been
 					built on this page; note that it does
@@ -850,6 +848,12 @@ struct buf_block_struct{
 	ulint		curr_side;	/* BTR_SEARCH_LEFT_SIDE or
 					BTR_SEARCH_RIGHT_SIDE in hash
 					indexing */
+	dict_index_t*	index;		/* Index for which the adaptive
+					hash index has been created.
+					This field may only be modified
+					while holding an s-latch or x-latch
+					on block->lock and an x-latch on
+					btr_search_latch. */
 	/* 6. Debug fields */
 #ifdef UNIV_SYNC_DEBUG
 	rw_lock_t	debug_latch;	/* in the debug version, each thread

innobase/include/dict0dict.h

@@ -197,7 +197,8 @@ dict_foreign_add_to_cache(
 /*======================*/
 					/* out: DB_SUCCESS or error code */
 	dict_foreign_t*	foreign,	/* in, own: foreign key constraint */
-	ibool		check_types);	/* in: TRUE=check type compatibility */
+	ibool		check_charsets);/* in: TRUE=check charset
+					compatibility */
 /*************************************************************************
 Checks if a table is referenced by foreign keys. */
 

innobase/include/dict0load.h

@@ -82,7 +82,8 @@ dict_load_foreigns(
 /*===============*/
 					/* out: DB_SUCCESS or error code */
 	const char*	table_name,	/* in: table name */
-	ibool		check_types);	/* in: TRUE=check type compatibility */
+	ibool		check_charsets);/* in: TRUE=check charsets
+					compatibility */
 /************************************************************************
 Prints to the standard output information on all tables found in the data
 dictionary system table. */

innobase/include/rem0cmp.h

@@ -24,7 +24,8 @@ cmp_types_are_equal(
 				/* out: TRUE if the types are considered
 				equal in comparisons */
 	dtype_t*	type1,	/* in: type 1 */
-	dtype_t*	type2);	/* in: type 2 */
+	dtype_t*	type2,	/* in: type 2 */
+	ibool		check_charsets); /* in: whether to check charsets */
 /*****************************************************************
 This function is used to compare two data fields for which we know the
 data type. */

innobase/rem/rem0cmp.c

@@ -99,7 +99,8 @@ cmp_types_are_equal(
 				/* out: TRUE if the types are considered
 				equal in comparisons */
 	dtype_t*	type1,	/* in: type 1 */
-	dtype_t*	type2)	/* in: type 2 */
+	dtype_t*	type2,	/* in: type 2 */
+	ibool		check_charsets) /* in: whether to check charsets */
 {
 	if (dtype_is_non_binary_string_type(type1->mtype, type1->prtype)
 	    && dtype_is_non_binary_string_type(type2->mtype, type2->prtype)) {
@@ -107,12 +108,12 @@ cmp_types_are_equal(
 		/* Both are non-binary string types: they can be compared if
 		and only if the charset-collation is the same */
 
-		if (dtype_get_charset_coll(type1->prtype)
-				== dtype_get_charset_coll(type2->prtype)) {
+		if (check_charsets) {
+			return(dtype_get_charset_coll(type1->prtype)
+			    == dtype_get_charset_coll(type2->prtype));
+		} else {
 			return(TRUE);
 		}
-
-		return(FALSE);
         }
 
 	if (dtype_is_binary_string_type(type1->mtype, type1->prtype)

innobase/row/row0mysql.c

@@ -2132,7 +2132,7 @@ row_table_add_foreign_constraints(
 
 	if (err == DB_SUCCESS) {
 		/* Check that also referencing constraints are ok */
-		err = dict_load_foreigns(name, trx->check_foreigns);
+		err = dict_load_foreigns(name, TRUE);
 	}
 
 	if (err != DB_SUCCESS) {
@@ -3590,7 +3590,8 @@ row_rename_table_for_mysql(
 	mem_heap_t*	heap			= NULL;
 	const char**	constraints_to_drop	= NULL;
 	ulint		n_constraints_to_drop	= 0;
-        ibool           recovering_temp_table   = FALSE;
+	ibool           recovering_temp_table   = FALSE;
+	ibool		old_is_tmp, new_is_tmp;
 	ulint		len;
 	ulint		i;
         ibool		success;
@@ -3630,6 +3631,9 @@ row_rename_table_for_mysql(
 	trx->op_info = "renaming table";
 	trx_start_if_not_started(trx);
 
+	old_is_tmp = row_is_mysql_tmp_table_name(old_name);
+	new_is_tmp = row_is_mysql_tmp_table_name(new_name);
+	
 	if (row_mysql_is_recovered_tmp_table(new_name)) {
 
                 recovering_temp_table = TRUE;
@@ -3676,7 +3680,7 @@ row_rename_table_for_mysql(
 	len = (sizeof str1) + (sizeof str2) + (sizeof str3) + (sizeof str5) - 4
 		+ ut_strlenq(new_name, '\'') + ut_strlenq(old_name, '\'');
 
-	if (row_is_mysql_tmp_table_name(new_name)) {
+	if (new_is_tmp) {
 		db_name_len = dict_get_db_name_len(old_name) + 1;
 
 		/* MySQL is doing an ALTER TABLE command and it renames the
@@ -3829,7 +3833,7 @@ row_rename_table_for_mysql(
 		the table is stored in a single-table tablespace */
 
 		success = dict_table_rename_in_cache(table, new_name,
-				!row_is_mysql_tmp_table_name(new_name));
+				!new_is_tmp);
 		if (!success) {
 			trx->error_state = DB_SUCCESS;
 			trx_general_rollback_for_mysql(trx, FALSE, NULL);
@@ -3846,19 +3850,16 @@ row_rename_table_for_mysql(
 			goto funct_exit;
 		}
 
-		err = dict_load_foreigns(new_name, trx->check_foreigns);
-
-		if (row_is_mysql_tmp_table_name(old_name)) {
+		/* We only want to switch off some of the type checking in
+		an ALTER, not in a RENAME. */
+		
+		err = dict_load_foreigns(new_name,
+			old_is_tmp ? trx->check_foreigns : TRUE);
 
-			/* MySQL is doing an ALTER TABLE command and it
-			renames the created temporary table to the name
-			of the original table. In the ALTER TABLE we maybe
-			created some FOREIGN KEY constraints for the temporary
-			table. But we want to load also the foreign key
-			constraint definitions for the original table name. */
+		if (err != DB_SUCCESS) {
+			ut_print_timestamp(stderr);
 
-			if (err != DB_SUCCESS) {
-	    			ut_print_timestamp(stderr);
+			if (old_is_tmp) {
 				fputs("  InnoDB: Error: in ALTER TABLE ",
 					stderr);
 				ut_print_name(stderr, trx, new_name);
@@ -3866,36 +3867,23 @@ row_rename_table_for_mysql(
 	"InnoDB: has or is referenced in foreign key constraints\n"
 	"InnoDB: which are not compatible with the new table definition.\n",
 					stderr);
-
-				ut_a(dict_table_rename_in_cache(table,
-					old_name, FALSE));
-				trx->error_state = DB_SUCCESS;
-				trx_general_rollback_for_mysql(trx, FALSE,
-									NULL);
-				trx->error_state = DB_SUCCESS;
-			}
-		} else {
-			if (err != DB_SUCCESS) {
-
-	    			ut_print_timestamp(stderr);
-
+			} else {
 				fputs(
 				"  InnoDB: Error: in RENAME TABLE table ",
 					stderr);
 				ut_print_name(stderr, trx, new_name);
 				fputs("\n"
-     "InnoDB: is referenced in foreign key constraints\n"
-     "InnoDB: which are not compatible with the new table definition.\n",
+	"InnoDB: is referenced in foreign key constraints\n"
+	"InnoDB: which are not compatible with the new table definition.\n",
 					stderr);
-     
-				ut_a(dict_table_rename_in_cache(table,
-					old_name, FALSE));
-						
-				trx->error_state = DB_SUCCESS;
-				trx_general_rollback_for_mysql(trx, FALSE,
-									NULL);
-				trx->error_state = DB_SUCCESS;
 			}
+
+			ut_a(dict_table_rename_in_cache(table,
+					old_name, FALSE));
+			trx->error_state = DB_SUCCESS;
+			trx_general_rollback_for_mysql(trx, FALSE,
+				NULL);
+			trx->error_state = DB_SUCCESS;
 		}
 	}
 funct_exit:	

mysql-test/r/innodb.result

@@ -2437,7 +2437,9 @@ a	b
 20	NULL
 drop table t1;
 create table t1 (v varchar(65530), key(v));
-ERROR HY000: Can't create table './test/t1' (errno: 139)
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+drop table t1;
 create table t1 (v varchar(65536));
 Warnings:
 Note	1246	Converting column 'v' from VARCHAR to TEXT
@@ -2577,22 +2579,49 @@ create table t8 (col1 blob, index(col1(767)))
 character set = latin1 engine = innodb;
 create table t9 (col1 varchar(512), col2 varchar(512), index(col1, col2))
 character set = latin1 engine = innodb;
+show create table t9;
+Table	Create Table
+t9	CREATE TABLE `t9` (
+  `col1` varchar(512) default NULL,
+  `col2` varchar(512) default NULL,
+  KEY `col1` (`col1`,`col2`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
 drop table t1, t2, t3, t4, t5, t6, t7, t8, t9;
-create table t1 (col1 varchar(768), index (col1))
+create table t1 (col1 varchar(768), index(col1))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t1.frm' (errno: 139)
-create table t2 (col1 varchar(768) primary key)
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t2 (col1 varbinary(768), index(col1))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t2.frm' (errno: 139)
-create table t3 (col1 varbinary(768) primary key)
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t3 (col1 text, index(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t3.frm' (errno: 139)
-create table t4 (col1 text, index(col1(768)))
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t4 (col1 blob, index(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t4.frm' (errno: 139)
-create table t5 (col1 blob, index(col1(768)))
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `col1` varchar(768) default NULL,
+  KEY `col1` (`col1`(767))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+drop table t1, t2, t3, t4;
+create table t1 (col1 varchar(768) primary key)
+character set = latin1 engine = innodb;
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t2 (col1 varbinary(768) primary key)
+character set = latin1 engine = innodb;
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t3 (col1 text, primary key(col1(768)))
+character set = latin1 engine = innodb;
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t4 (col1 blob, primary key(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t5.frm' (errno: 139)
+ERROR 42000: Specified key was too long; max key length is 767 bytes
 CREATE TABLE t1
 (
 id INT PRIMARY KEY
@@ -2772,6 +2801,38 @@ insert into t2 values (4,_ucs2 0x05612020,_ucs2 0x05612020,'taken');
 drop table t1;
 drop table t2;
 commit;
+set foreign_key_checks=0;
+create table t2 (a int primary key, b int, foreign key (b) references t1(a)) engine = innodb;
+create table t1(a char(10) primary key, b varchar(20)) engine = innodb;
+ERROR HY000: Can't create table './test/t1.frm' (errno: 150)
+set foreign_key_checks=1;
+drop table t2;
+set foreign_key_checks=0;
+create table t1(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=latin1;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=utf8;
+ERROR HY000: Can't create table './test/t2.frm' (errno: 150)
+set foreign_key_checks=1;
+drop table t1;
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb;
+create table t1(a varchar(10) primary key) engine = innodb;
+alter table t1 modify column a int;
+Got one of the listed errors
+set foreign_key_checks=1;
+drop table t2,t1;
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=latin1;
+create table t1(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=latin1;
+alter table t1 convert to character set utf8;
+set foreign_key_checks=1;
+drop table t2,t1;
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=latin1;
+create table t3(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=utf8;
+rename table t3 to t1;
+ERROR HY000: Error on rename of './test/t3' to './test/t1' (errno: 150)
+set foreign_key_checks=1;
+drop table t2,t3;
 create table t1 (a varchar(255) character set utf8,
 b varchar(255) character set utf8,
 c varchar(255) character set utf8,
@@ -2785,4 +2846,3 @@ d varchar(255) character set utf8,
 e varchar(255) character set utf8,
 key (a,b,c,d,e)) engine=innodb;
 ERROR 42000: Specified key was too long; max key length is 3072 bytes
-End of 5.0 tests

mysql-test/t/innodb.test

@@ -1356,8 +1356,8 @@ source include/varchar.inc;
 # Clean up filename -- embedded server reports whole path without .frm,
 # regular server reports relative path with .frm (argh!)
 --replace_result \\ / $MYSQL_TEST_DIR . /var/master-data/ / t1.frm t1
---error 1005
 create table t1 (v varchar(65530), key(v));
+drop table t1;
 create table t1 (v varchar(65536));
 show create table t1;
 drop table t1;
@@ -1485,7 +1485,7 @@ CREATE TEMPORARY TABLE t2
 DROP TABLE t1;
 
 #
-# Test that index column max sizes are checked (bug #13315)
+# Test that index column max sizes are honored (bug #13315)
 #
 
 # prefix index
@@ -1512,22 +1512,36 @@ create table t8 (col1 blob, index(col1(767)))
 create table t9 (col1 varchar(512), col2 varchar(512), index(col1, col2))
  character set = latin1 engine = innodb;
 
+show create table t9;
+
 drop table t1, t2, t3, t4, t5, t6, t7, t8, t9;
 
---error 1005
-create table t1 (col1 varchar(768), index (col1))
+# these should have their index length trimmed
+create table t1 (col1 varchar(768), index(col1))
  character set = latin1 engine = innodb;
---error 1005
-create table t2 (col1 varchar(768) primary key)
+create table t2 (col1 varbinary(768), index(col1))
  character set = latin1 engine = innodb;
---error 1005
-create table t3 (col1 varbinary(768) primary key)
+create table t3 (col1 text, index(col1(768)))
  character set = latin1 engine = innodb;
---error 1005
-create table t4 (col1 text, index(col1(768)))
+create table t4 (col1 blob, index(col1(768)))
  character set = latin1 engine = innodb;
---error 1005
-create table t5 (col1 blob, index(col1(768)))
+
+show create table t1;
+
+drop table t1, t2, t3, t4;
+
+# these should be refused
+--error 1071
+create table t1 (col1 varchar(768) primary key)
+ character set = latin1 engine = innodb;
+--error 1071
+create table t2 (col1 varbinary(768) primary key)
+ character set = latin1 engine = innodb;
+--error 1071
+create table t3 (col1 text, primary key(col1(768)))
+ character set = latin1 engine = innodb;
+--error 1071
+create table t4 (col1 blob, primary key(col1(768)))
  character set = latin1 engine = innodb;
 
 #
@@ -1752,6 +1766,56 @@ drop table t1;
 drop table t2;
 commit;
 
+# tests for bugs #9802 and #13778
+
+# test that FKs between invalid types are not accepted
+
+set foreign_key_checks=0;
+create table t2 (a int primary key, b int, foreign key (b) references t1(a)) engine = innodb;
+-- error 1005
+create table t1(a char(10) primary key, b varchar(20)) engine = innodb;
+set foreign_key_checks=1;
+drop table t2;
+
+# test that FKs between different charsets are not accepted in CREATE even
+# when f_k_c is 0
+
+set foreign_key_checks=0;
+create table t1(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=latin1;
+-- error 1005
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=utf8;
+set foreign_key_checks=1;
+drop table t1;
+
+# test that invalid datatype conversions with ALTER are not allowed
+
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb;
+create table t1(a varchar(10) primary key) engine = innodb;
+-- error 1025,1025
+alter table t1 modify column a int;
+set foreign_key_checks=1;
+drop table t2,t1;
+
+# test that charset conversions with ALTER are allowed when f_k_c is 0
+
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=latin1;
+create table t1(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=latin1;
+alter table t1 convert to character set utf8;
+set foreign_key_checks=1;
+drop table t2,t1;
+
+# test that RENAME does not allow invalid charsets when f_k_c is 0
+
+set foreign_key_checks=0;
+create table t2 (a varchar(10), foreign key (a) references t1(a)) engine = innodb DEFAULT CHARSET=latin1;
+create table t3(a varchar(10) primary key) engine = innodb DEFAULT CHARSET=utf8;
+-- error 1025
+rename table t3 to t1;
+set foreign_key_checks=1;
+drop table t2,t3;
+
 #
 # Test that we can create a large (>1K) key
 #

sql/ha_innodb.cc

@@ -2519,6 +2519,12 @@ ha_innobase::open(
   	DBUG_RETURN(0);
 }
 
+uint
+ha_innobase::max_supported_key_part_length() const
+{
+	return(DICT_MAX_INDEX_COL_LEN - 1);
+}
+
 /**********************************************************************
 Closes a handle to an InnoDB table. */
 
@@ -4675,6 +4681,9 @@ create_index(
 				0, prefix_len);
 	}
 
+	/* Even though we've defined max_supported_key_part_length, we
+	still do our own checking using field_lengths to be absolutely
+	sure we don't create too long indexes. */
 	error = row_create_index_for_mysql(index, trx, field_lengths);
 
 	error = convert_error_code_to_mysql(error, NULL);

sql/ha_innodb.h

@@ -110,7 +110,7 @@ class ha_innobase: public handler
 				but currently MySQL does not work with keys
 				whose size is > MAX_KEY_LENGTH */
   	uint max_supported_key_length() const { return 3500; }
-  	uint max_supported_key_part_length() const { return 3500; }
+  	uint max_supported_key_part_length() const;
 	const key_map *keys_to_use_for_scanning() { return &key_map_full; }
   	bool has_transactions()  { return 1;}