Bug#17877 - Corrupted spatial index

CHECK TABLE could complain about a fully intact spatial index.
A wrong comparison operator was used for table checking. 
The result was that it checked for non-matching spatial keys. 
This succeeded if at least two different keys were present, 
but failed if only the matching key was present.

I fixed the key comparison.

myisam/mi_check.c

@@ -1155,12 +1155,13 @@ int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
 	      */
               int search_result= (keyinfo->flag & HA_SPATIAL) ?
                 rtree_find_first(info, key, info->lastkey, key_length,
-                                 SEARCH_SAME) : 
+                                 MBR_EQUAL | MBR_DATA) : 
                 _mi_search(info,keyinfo,info->lastkey,key_length,
                            SEARCH_SAME, info->s->state.key_root[key]);
               if (search_result)
               {
-                mi_check_print_error(param,"Record at: %10s  Can't find key for index: %2d",
+                mi_check_print_error(param,"Record at: %10s  "
+                                     "Can't find key for index: %2d",
                                      llstr(start_recpos,llbuff),key+1);
                 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                   goto err2;

myisam/mi_key.c

@@ -54,7 +54,7 @@ uint _mi_make_key(register MI_INFO *info, uint keynr, uchar *key,
       TODO: nulls processing
     */
 #ifdef HAVE_SPATIAL
-    return sp_make_key(info,keynr,key,record,filepos);
+    DBUG_RETURN(sp_make_key(info,keynr,key,record,filepos));
 #else
     DBUG_ASSERT(0); /* mi_open should check that this never happens*/
 #endif

myisam/rt_index.c

@@ -183,9 +183,11 @@ int rtree_find_first(MI_INFO *info, uint keynr, uchar *key, uint key_length,
     return -1;
   }
 
-  /* Save searched key */
-  memcpy(info->first_mbr_key, key, keyinfo->keylength -
-	 info->s->base.rec_reflength);
+  /*
+    Save searched key, include data pointer.
+    The data pointer is required if the search_flag contains MBR_DATA.
+  */
+  memcpy(info->first_mbr_key, key, keyinfo->keylength);
   info->last_rkey_length = key_length;
 
   info->rtree_recursion_depth = -1;

myisam/rt_mbr.c

@@ -52,10 +52,14 @@
     if (EQUAL_CMP(amin, amax, bmin, bmax)) \
       return 1; \
   } \
-  else /* if (nextflag & MBR_DISJOINT) */ \
+  else if (nextflag & MBR_DISJOINT) \
   { \
     if (DISJOINT_CMP(amin, amax, bmin, bmax)) \
       return 1; \
+  }\
+  else /* if unknown comparison operator */ \
+  { \
+    DBUG_ASSERT(0); \
   }
 
 #define RT_CMP_KORR(type, korr_func, len, nextflag) \

mysql-test/r/gis-rtree.result

@@ -817,3 +817,43 @@ check table t1 extended;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
+CREATE TABLE t1 (
+c1 geometry NOT NULL default '',
+SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+CHECK TABLE t1 EXTENDED;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
+CREATE TABLE t1 (
+c1 geometry NOT NULL default '',
+SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-65.7402776999 -96.6686111000,
+                            -65.7372222000 -96.5516666000,
+                            -65.8502777000 -96.5461111000,
+                            -65.8527777000 -96.6627777000,
+                            -65.7402776999 -96.6686111000))'));
+INSERT INTO t1 (c1) VALUES (
+PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+CHECK TABLE t1 EXTENDED;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;

mysql-test/t/gis-rtree.test

@@ -188,4 +188,48 @@ check table t1 extended;
 
 drop table t1;
 
+#
+# Bug#17877 - Corrupted spatial index
+#
+CREATE TABLE t1 (
+  c1 geometry NOT NULL default '',
+  SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+# This showed a missing key.
+CHECK TABLE t1 EXTENDED;
+DROP TABLE t1;
+#
+CREATE TABLE t1 (
+  c1 geometry NOT NULL default '',
+  SPATIAL KEY i1 (c1(32))
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-65.7402776999 -96.6686111000,
+                            -65.7372222000 -96.5516666000,
+                            -65.8502777000 -96.5461111000,
+                            -65.8527777000 -96.6627777000,
+                            -65.7402776999 -96.6686111000))'));
+# This is the same as the first insert to get a non-unique key.
+INSERT INTO t1 (c1) VALUES (
+  PolygonFromText('POLYGON((-18.6086111000 -66.9327777000,
+                            -18.6055555000 -66.8158332999,
+                            -18.7186111000 -66.8102777000,
+                            -18.7211111000 -66.9269443999,
+                            -18.6086111000 -66.9327777000))'));
+# This showed (and still shows) OK.
+CHECK TABLE t1 EXTENDED;
+DROP TABLE t1;
+
 # End of 4.1 tests