Fixed MDEV-4291: Assertion `trid >= info->s->state.create_trid' failure or...

Fixed MDEV-4291: Assertion `trid >= info->s->state.create_trid' failure or data corruption (key points to record
outside datafile) on INSERT into an Aria table.

The isssue was that the check if a table was moved between systems didn't take into account that create_trid could be bigger than the current max trid on the new system.
This could only happen if one tried to move a table that one had just done a 'REPAIR TABLE' on.
Tables that one had run 'aria_chk --zerofill' on worked.

Fixed this by assuming that if create_trid is too big then the table has been moved from one system to another and we have to do an automatic zerofill.

Other fixed:

- Added a check to detect a wrong create_trid in 'check table'.
- aria_chk -dvv will now write out also the create_trid (to make future error finding easier)
- aria_chk --zerofill doesn't anymore require a aria_control_file
- Removed some warnings from safemalloc when using aria_chk, ma_test1 and ma_test2.


include/myisamchk.h:
  Removed wrong 'QQ' flags (the flags are used by myisamchk and aria_chk)
storage/maria/ha_maria.cc:
  maria_chk_status() can now also return an error.
storage/maria/ma_check.c:
  In maria_chk_status() check if create_trid value is too big.
storage/maria/ma_open.c:
  Changed check if table is moved so that we can detect wrong create_trid values.
  Don't set STATE_NOT_MOVABLE flag if we are doing repair/check. This was done so that aria_chk can print out the movable flag.
storage/maria/ma_test1.c:
  Added code to suppress memory leaks from safemalloc
storage/maria/ma_test2.c:
  Added code to suppress memory leaks from safemalloc
storage/maria/maria_chk.c:
  Added code to suppress memory leaks from safemalloc.
  Make help text a bit better for --HELP and --zerofill.
  Incresed version number.
  Don't require a control file if we are only doing --zerofill
  Print out 'create_trid' when doing --describe --verbose
storage/maria/unittest/ma_test_recovery.expected:
  Updated result file

include/myisamchk.h

@@ -31,27 +31,27 @@
 #define T_AUTO_REPAIR           2		/* QQ to be removed */
 #define T_BACKUP_DATA           4
 #define T_CALC_CHECKSUM         8
-#define T_CHECK                 16		/* QQ to be removed */
-#define T_CHECK_ONLY_CHANGED    32		/* QQ to be removed */
+#define T_CHECK                 16
+#define T_CHECK_ONLY_CHANGED    32
 #define T_CREATE_MISSING_KEYS   64
 #define T_DESCRIPT              128
 #define T_DONT_CHECK_CHECKSUM   256
 #define T_EXTEND                512
-#define T_FAST                  (1L << 10)	/* QQ to be removed */
-#define T_FORCE_CREATE          (1L << 11)	/* QQ to be removed */
+#define T_FAST                  (1L << 10)
+#define T_FORCE_CREATE          (1L << 11)
 #define T_FORCE_UNIQUENESS      (1L << 12)
 #define T_INFO                  (1L << 13)
 #define T_MEDIUM                (1L << 14)
-#define T_QUICK                 (1L << 15)	/* QQ to be removed */
-#define T_READONLY              (1L << 16)	/* QQ to be removed */
+#define T_QUICK                 (1L << 15)
+#define T_READONLY              (1L << 16)
 #define T_REP                   (1L << 17)
-#define T_REP_BY_SORT           (1L << 18)	/* QQ to be removed */
-#define T_REP_PARALLEL          (1L << 19)	/* QQ to be removed */
+#define T_REP_BY_SORT           (1L << 18)
+#define T_REP_PARALLEL          (1L << 19)
 #define T_RETRY_WITHOUT_QUICK   (1L << 20)
 #define T_SAFE_REPAIR           (1L << 21)
 #define T_SILENT                (1L << 22)
-#define T_SORT_INDEX            (1L << 23)	/* QQ to be removed */
-#define T_SORT_RECORDS          (1L << 24)	/* QQ to be removed */
+#define T_SORT_INDEX            (1L << 23)
+#define T_SORT_RECORDS          (1L << 24)
 #define T_STATISTICS            (1L << 25)
 #define T_UNPACK                (1L << 26)
 #define T_UPDATE_STATE          (1L << 27)

storage/maria/ha_maria.cc

@@ -1309,8 +1309,9 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt)
 
   old_proc_info= thd_proc_info(thd, "Checking status");
   thd_progress_init(thd, 3);
-  (void) maria_chk_status(&param, file);                // Not fatal
-  error= maria_chk_size(&param, file);
+  error= maria_chk_status(&param, file);                // Not fatal
+  if (maria_chk_size(&param, file))
+    error= 1;
   if (!error)
     error|= maria_chk_del(&param, file, param.testflag);
   thd_proc_info(thd, "Checking keys");

storage/maria/ma_check.c

@@ -140,13 +140,22 @@ void maria_chk_init_for_check(HA_CHECK *param, MARIA_HA *info)
     Set up transaction handler so that we can see all rows. When rows is read
     we will check the found id against param->max_tried
   */
-  if (param->max_trid == 0)
+  if (!info->s->base.born_transactional)
+  {
+    /*
+      There are no trids. Howver we want to set max_trid to make test of
+      create_trid simpler.
+    */
+    param->max_trid= ~(TrID) 0;
+  }
+  else if (param->max_trid == 0)
   {
     if (!ma_control_file_inited())
       param->max_trid= 0;      /* Give warning for first trid found */
     else
       param->max_trid= max_trid_in_system();
   }
+
   maria_ignore_trids(info);
 }
 
@@ -179,6 +188,13 @@ int maria_chk_status(HA_CHECK *param, MARIA_HA *info)
     if (param->testflag & T_UPDATE_STATE)
       param->warning_printed=save;
   }
+  if (share->state.create_trid > param->max_trid)
+  {
+    _ma_check_print_warning(param,
+                            "Table create_trd (%llu) > current max_transaction id (%llu).  Table needs to be repaired or zerofilled to be usable",
+                            share->state.create_trid, param->max_trid);
+    return 1;
+  }
   return 0;
 }
 
@@ -3552,7 +3568,10 @@ int maria_zerofill(HA_CHECK *param, MARIA_HA *info, const char *name)
     /* Ensure state is later flushed to disk, if within maria_chk */
     info->update= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
 
-    /* Reset create_trid to make file comparable */
+    /*
+      Reset create_trid to make file comparable and to ensure that new
+      trid's in the file starts from 0.
+    */
     share->state.create_trid= 0;
   }
   if (reenable_logging)

storage/maria/ma_open.c

@@ -437,15 +437,23 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
       share->open_count_not_zero_on_open= 1;
 
     /*
+      A transactional table is not usable on this system if:
+      - share->state.create_trid > trnman_get_max_trid()
+        - Critical as trid as stored releativel to create_trid.
+      - uuid is different
+      
+        STATE_NOT_MOVABLE is reset when a table is zerofilled
+        (has no LSN's and no trids)
+
       We can ignore testing uuid if STATE_NOT_MOVABLE is set, as in this
-      case the uuid will be set in _ma_mark_file_changed()
+      case the uuid will be set in _ma_mark_file_changed().
     */
-    if ((share->state.changed & STATE_NOT_MOVABLE) &&
-        share->base.born_transactional &&
-        ((!(open_flags & HA_OPEN_IGNORE_MOVED_STATE) &&
-          memcmp(share->base.uuid, maria_uuid, MY_UUID_SIZE)) ||
-         (share->state.create_trid > trnman_get_max_trid() &&
-          !maria_in_recovery)))
+    if (share->base.born_transactional &&
+        ((share->state.create_trid > trnman_get_max_trid() &&
+         !maria_in_recovery) ||
+         ((share->state.changed & STATE_NOT_MOVABLE) &&
+          ((!(open_flags & HA_OPEN_IGNORE_MOVED_STATE) &&
+            memcmp(share->base.uuid, maria_uuid, MY_UUID_SIZE))))))
     {
       DBUG_PRINT("warning", ("table is moved from another system.  uuid_diff: %d  create_trid: %lu  max_trid: %lu",
                             memcmp(share->base.uuid, maria_uuid,
@@ -756,7 +764,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
                   HA_ERR_CRASHED_ON_REPAIR : HA_ERR_CRASHED_ON_USAGE);
         goto err;
       }
-      else
+      else if (!(open_flags & HA_OPEN_FOR_REPAIR))
       {
         /* create_rename_lsn != LSN_NEEDS_NEW_STATE_LSNS */
         share->state.changed|= STATE_NOT_MOVABLE;

storage/maria/ma_test1.c

@@ -55,7 +55,6 @@ static void create_key(uchar *key,uint rownr);
 static void create_record(uchar *record,uint rownr);
 static void update_record(uchar *record);
 
-
 /*
   These are here only for testing of recovery with undo. We are not
   including maria_def.h here as this test is also to be an example of
@@ -506,6 +505,7 @@ end:
       break;
     }
     printf("Dying on request without maria_commit()/maria_close()\n");
+    sf_leaking_memory= 1;
     exit(0);
   }
 
@@ -514,6 +514,7 @@ end:
   if (maria_close(file))
     goto err;
   maria_end();
+  my_uuid_end();
   my_end(MY_CHECK_ERROR);
 
   return (0);

storage/maria/ma_test2.c

@@ -971,6 +971,7 @@ end:
       break;
     }
     printf("Dying on request without maria_commit()/maria_close()\n");
+    sf_leaking_memory= 1; /* no memory leak reports here */
     exit(0);
   }
   if (maria_commit(file))
@@ -1017,6 +1018,7 @@ reads:      %10lu\n",
   }
   maria_end();
   my_free(blob_buffer);
+  my_uuid_end();
   my_end(silent ? MY_CHECK_ERROR : MY_CHECK_ERROR | MY_GIVE_INFO);
   return(0);
 err:
@@ -1029,6 +1031,8 @@ err2:
     maria_close(file);
   }
   maria_end();
+  my_uuid_end();
+  my_end(0);
   return(1);
 } /* main */
 

storage/maria/maria_chk.c

@@ -88,9 +88,22 @@ static int sort_record_index(MARIA_SORT_PARAM *sort_param, MARIA_PAGE *page,
 			     uint sortkey, File new_file,
                              my_bool update_index);
 static my_bool write_log_record(HA_CHECK *param);
+static void my_exit(int exit_code) __attribute__ ((noreturn));
 
 HA_CHECK check_param;
 
+/* Free memory and exit */
+
+static void my_exit(int exit_code)
+{
+  free_tmpdir(&maria_chk_tmpdir);
+  free_defaults(default_argv);
+  my_end(check_param.testflag & T_INFO ?
+         MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR);
+  exit(exit_code);
+}
+  
+
 	/* Main program */
 
 int main(int argc, char **argv)
@@ -178,12 +191,8 @@ end:
     printf("\nTotal of all %d Aria-files:\nData records: %9s   Deleted blocks: %9s\n",check_param.total_files,llstr(check_param.total_records,buff),
 	   llstr(check_param.total_deleted,buff2));
   }
-  free_defaults(default_argv);
-  free_tmpdir(&maria_chk_tmpdir);
   maria_end();
-  my_end(check_param.testflag & T_INFO ?
-         MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR);
-  exit(error);
+  my_exit(error);
 #ifndef _lint
   return 0;				/* No compiler warning */
 #endif
@@ -252,10 +261,10 @@ static struct my_option my_long_options[] =
    "Restart with -r if there are any errors in the table. States will be updated as with --update-state.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"HELP", 'H',
-   "Display this help and exit.",
+   "Print all argument options sorted alphabetically and exit.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"help", '?',
-   "Display this help and exit.",
+   "Print all options by groups and exit. See also --HELP",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"information", 'i',
    "Print statistics information about table that is checked.",
@@ -406,7 +415,7 @@ static struct my_option my_long_options[] =
     (char**) &maria_stats_method_str, (char**) &maria_stats_method_str, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   { "zerofill", 'z',
-    "Fill empty space in data and index files with zeroes,",
+    "Fill empty space in data and index files with zeroes. This makes the data file movable between different servers.",
     0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   { "zerofill-keep-lsn", OPT_ZEROFILL_KEEP_LSN,
     "Like --zerofill but does not zero out LSN of data/index pages;"
@@ -418,7 +427,7 @@ static struct my_option my_long_options[] =
 
 static void print_version(void)
 {
-  printf("%s  Ver 1.1 for %s at %s\n", my_progname, SYSTEM_TYPE,
+  printf("%s  Ver 1.2 for %s at %s\n", my_progname, SYSTEM_TYPE,
 	 MACHINE_TYPE);
 }
 
@@ -437,8 +446,8 @@ static void usage(void)
   -#, --debug=...     Output debug log. Often this is 'd:t:o,filename'.\n");
 #endif
   printf("\
-  -H, --HELP          Display this help and exit.\n\
-  -?, --help          Display this help and exit.\n\
+  -H, --HELP          Print all argument options sorted alphabetically.\n\
+  -?, --help          Print all options by groups\n\
   --datadir=path      Path for control file (and logs if --logdir not used)\n\
   --logdir=path       Path for log files\n\
   --ignore-control-file  Don't open the control file. Only use this if you\n\
@@ -554,7 +563,9 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
 		      (It may be VERY slow to do a sort the first time!).\n\
   -b,  --block-search=#\n\
                       Find a record, a block at given offset belongs to.\n\
-  -z,  --zerofill     Fill empty space in data and index files with zeroes\n\
+  -z,  --zerofill     Fill empty space in data and index files with zeroes.\n\
+                      This makes the data file movable between different \n\
+                      servers.\n\
   --zerofill-keep-lsn Like --zerofill but does not zero out LSN of\n\
                       data/index pages.");
 
@@ -757,7 +768,7 @@ get_one_option(int optid,
 	fprintf(stderr,
 		"The value of the sort key is bigger than max key: %d.\n",
 		MARIA_MAX_KEY);
-	exit(1);
+	my_exit(1);
       }
     }
     break;
@@ -785,7 +796,7 @@ get_one_option(int optid,
     break;
   case 'V':
     print_version();
-    exit(0);
+    my_exit(0);
   case OPT_CORRECT_CHECKSUM:
     if (argument == disabled_my_option)
       check_param.testflag&= ~T_CALC_CHECKSUM;
@@ -800,7 +811,7 @@ get_one_option(int optid,
     if ((method=find_type(argument, &maria_stats_method_typelib, 2)) <= 0)
     {
       fprintf(stderr, "Invalid value of stats_method: %s.\n", argument);
-      exit(1);
+      my_exit(1);
     }
     switch (method-1) {
     case 0:
@@ -836,10 +847,11 @@ get_one_option(int optid,
     break;
   case 'H':
     my_print_help(my_long_options);
-    exit(0);
+    my_print_variables(my_long_options);
+    my_exit(0);
   case '?':
     usage();
-    exit(0);
+    my_exit(0);
   }
   return 0;
 }
@@ -856,7 +868,7 @@ static void get_options(register int *argc,register char ***argv)
     check_param.testflag|=T_WRITE_LOOP;
 
   if ((ho_error=handle_options(argc, argv, my_long_options, get_one_option)))
-    exit(ho_error);
+    my_exit(ho_error);
 
   /* If using repair, then update checksum if one uses --update-state */
   if ((check_param.testflag & T_UPDATE_STATE) &&
@@ -866,7 +878,7 @@ static void get_options(register int *argc,register char ***argv)
   if (*argc == 0)
   {
     usage();
-    exit(-1);
+    my_exit(-1);
   }
 
   if ((check_param.testflag & T_UNPACK) &&
@@ -874,7 +886,7 @@ static void get_options(register int *argc,register char ***argv)
   {
     fprintf(stderr, "%s: --unpack can't be used with --quick or --sort-records\n",
 		 my_progname_short);
-    exit(1);
+    my_exit(1);
   }
   if ((check_param.testflag & T_READONLY) &&
       (check_param.testflag &
@@ -883,7 +895,7 @@ static void get_options(register int *argc,register char ***argv)
   {
     fprintf(stderr, "%s: Can't use --readonly when repairing or sorting\n",
 		 my_progname_short);
-    exit(1);
+    my_exit(1);
   }
 
   if (!opt_debug)
@@ -891,20 +903,26 @@ static void get_options(register int *argc,register char ***argv)
     DEBUGGER_OFF;                               /* Speed up things a bit */
   }
   if (init_tmpdir(&maria_chk_tmpdir, opt_tmpdir))
-    exit(1);
+    my_exit(1);
 
   check_param.tmpdir=&maria_chk_tmpdir;
 
   if (set_collation_name)
     if (!(set_collation= get_charset_by_name(set_collation_name,
                                              MYF(MY_WME))))
-      exit(1);
+      my_exit(1);
 
   if (maria_data_root != default_log_dir && opt_log_dir == default_log_dir)
   {
     /* --datadir was used and --log-dir was not. Set log-dir to datadir */
     opt_log_dir= maria_data_root;
   }
+
+  /* If we are using zerofill, then we don't need to read the control file */
+  if ((check_param.testflag & (T_ZEROFILL_KEEP_LSN | T_ZEROFILL)) &&
+      !(check_param.testflag & ~(T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX | T_STATISTICS | T_CHECK | T_FAST | T_CHECK_ONLY_CHANGED)))
+    opt_ignore_control_file= 1;
+
   return;
 } /* get options */
 
@@ -1475,6 +1493,8 @@ static void descript(HA_CHECK *param, register MARIA_HA *info, char *name)
              LSN_IN_PARTS(share->state.create_rename_lsn),
              LSN_IN_PARTS(share->state.is_of_horizon),
              LSN_IN_PARTS(share->state.skip_redo_lsn));
+      printf("create_trid:         %s\n",
+             llstr(share->state.create_trid, llbuff));
     }
     compile_time_assert((MY_UUID_STRING_LENGTH + 1) <= sizeof(buff));
     buff[MY_UUID_STRING_LENGTH]= 0;