Skip to content

M
mariadb
Commit cc00660f authored by msvensson@shellback.(none)'s avatar msvensson@shellback.(none)
Browse files
Options

WL#3935 Improve mysqltest report and exitcode for diff 

 - Move the code to generate test report to the test tool(in this
   case mysqltest) where the best control of what failed is
- Simplify the code in mysql-test-run.pl
- mysqltest will now find what diff to use in a best effort attempt
  using "diff -u", "diff -c" and finally dumping the two files verbatim
  in the report.
parent d6f02500
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 468 additions and 201 deletions
client/mysqltest.c
View file @ cc00660f
This diff is collapsed.
mysql-test/lib/mtr_report.pl
View file @ cc00660f
......@@ -26,7 +26,6 @@ sub mtr_report_test_failed($);
sub mtr_report_test_skipped($);
sub mtr_report_test_not_skipped_though_disabled($);
sub mtr_show_failed_diff ($);
sub mtr_report_stats ($);
sub mtr_print_line ();
sub mtr_print_thick_line ();
......@@ -45,55 +44,6 @@ sub mtr_verbose (@);
#
##############################################################################
# We can't use diff -u or diff -a as these are not portable
sub mtr_show_failed_diff ($) {
my $result_file_name= shift;
# The reject and log files have been dumped to
# to filenames based on the result_file's name
my $tname= basename($result_file_name);
$tname=~ s/\..*$//;
my $reject_file= "r/$tname.reject";
my $result_file= "r/$tname.result";
my $log_file= "r/$tname.log";
my $eval_file= "r/$tname.eval";
if ( $::opt_suite ne "main" )
{
$reject_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$reject_file";
$result_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$result_file";
$eval_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$eval_file";
$log_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$log_file";
}
if ( -f $eval_file )
{
$result_file= $eval_file;
}
my $diffopts= $::opt_udiff ? "-u" : "-c";
if ( -f $reject_file )
{
print "Below are the diffs between actual and expected results:\n";
print "-------------------------------------------------------\n";
# FIXME check result code?!
mtr_run("diff",[$diffopts,$result_file,$reject_file], "", "", "", "");
print "-------------------------------------------------------\n";
print "Please follow the instructions outlined at\n";
print "http://www.mysql.com/doc/en/Reporting_mysqltest_bugs.html\n";
print "to find the reason to this problem and how to report this.\n\n";
}
if ( -f $log_file )
{
print "Result from queries before failure can be found in $log_file\n";
# FIXME Maybe a tail -f -n 10 $log_file here
}
}
sub mtr_report_test_name ($) {
my $tinfo= shift;
......@@ -165,16 +115,23 @@ sub mtr_report_test_failed ($) {
if ( $tinfo->{'comment'} )
{
# The test failure has been detected by mysql-test-run.pl
# when starting the servers or due to other error, the reason for
# failing the test is saved in "comment"
print "\nERROR: $tinfo->{'comment'}\n";
}
elsif ( -f $::path_timefile )
{
print "\nErrors are (from $::path_timefile) :\n";
# Test failure was detected by test tool and it's report
# about what failed has been saved to file. Display the report.
print "\n";
print mtr_fromfile($::path_timefile); # FIXME print_file() instead
print "\n(the last lines may be the most important ones)\n";
print "\n";
}
else
{
# Neither this script or the test tool has recorded info
# about why the test has failed. Should be debugged.
print "\nUnexpected termination, probably when starting mysqld\n";
}
}
......
mysql-test/mysql-test-run.pl
View file @ cc00660f
......@@ -83,7 +83,6 @@ require "lib/mtr_io.pl";
require "lib/mtr_gcov.pl";
require "lib/mtr_gprof.pl";
require "lib/mtr_report.pl";
require "lib/mtr_diff.pl";
require "lib/mtr_match.pl";
require "lib/mtr_misc.pl";
require "lib/mtr_stress.pl";
......@@ -281,8 +280,6 @@ our $opt_wait_for_slave;
our $opt_warnings;
our $opt_udiff;
our $opt_skip_ndbcluster= 0;
our $opt_skip_ndbcluster_slave= 0;
our $opt_with_ndbcluster= 0;
......@@ -619,7 +616,6 @@ sub command_line_setup () {
'start-dirty' => \$opt_start_dirty,
'start-and-exit' => \$opt_start_and_exit,
'timer!' => \$opt_timer,
'unified-diff|udiff' => \$opt_udiff,
'user=s' => \$opt_user,
'testcase-timeout=i' => \$opt_testcase_timeout,
'suite-timeout=i' => \$opt_suite_timeout,
......@@ -3603,7 +3599,6 @@ sub report_failure_and_restart ($) {
my $tinfo= shift;
mtr_report_test_failed($tinfo);
mtr_show_failed_diff($tinfo->{'result_file'});
print "\n";
if ( $opt_force )
{
......@@ -4788,6 +4783,9 @@ sub run_mysqltest ($) {
mtr_add_arg($args, "--test-file=%s", $tinfo->{'path'});
# Number of lines of resut to include in failure report
mtr_add_arg($args, "--tail-lines=20");
if ( defined $tinfo->{'result_file'} ) {
mtr_add_arg($args, "--result-file=%s", $tinfo->{'result_file'});
}
......@@ -5199,7 +5197,6 @@ Misc options
fast Don't try to clean up from earlier runs
reorder Reorder tests to get fewer server restarts
help Get this help text
unified-diff | udiff When presenting differences, use unified diff
testcase-timeout=MINUTES Max test case run time (default $default_testcase_timeout)
suite-timeout=MINUTES Max test suite run time (default $default_suite_timeout)
......
mysql-test/r/mysqltest.result
View file @ cc00660f
......@@ -461,7 +461,6 @@ root@localhost
--------------------------------------------------------------------------------
this will be executed
this will be executed
mysqltest: Result length mismatch
mysqltest: The test didn't produce any output
Failing multi statement query
mysqltest: At line 3: query 'create table t1 (a int primary key);
......@@ -473,6 +472,8 @@ mysqltest: At line 3: query 'create table t1 (a int primary key);
insert into t1 values (1);
select 'select-me';
insertz 'error query'' failed: 1064: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'insertz 'error query'' at line 1
More results from queries before failure can be found in MYSQLTEST_VARDIR/tmp/bug11731.log
drop table t1;
Multi statement using expected error
create table t1 (a int primary key);
......
mysql-test/t/mysqltest.test
View file @ cc00660f
......@@ -436,6 +436,8 @@ EOF
--error 1
--exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/mysqltest.sql 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
#
# Extra delimiter
#
......@@ -786,6 +788,7 @@ echo $var3_var3;
--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
--error 1
--exec echo "source $MYSQLTEST_VARDIR/tmp/recursive.sql;" | $MYSQL_TEST 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/recursive.sql;
# Source a file with error
--exec echo "garbage ;" > $MYSQLTEST_VARDIR/tmp/error.sql
......@@ -793,6 +796,7 @@ echo $var3_var3;
--error 1
--exec echo "source $MYSQLTEST_VARDIR/tmp/error.sql;" | $MYSQL_TEST 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/error.sql;
# Test execution of source in a while loop
--write_file $MYSQLTEST_VARDIR/tmp/sourced.inc
......@@ -1171,6 +1175,8 @@ EOF
--error 1
--exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/mysqltest.sql 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
# ----------------------------------------------------------------------------
# Test error messages returned from comments starting with a command
# ----------------------------------------------------------------------------
......@@ -1296,6 +1302,8 @@ EOF
--error 1
--exec echo "source $MYSQLTEST_VARDIR/tmp/mysqltest.sql;" | $MYSQL_TEST 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
# connect when "disable_abort_on_error" caused "connection not found"
--replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT
--disable_abort_on_error
......@@ -1399,7 +1407,11 @@ select "this will be executed";
--exec touch $MYSQLTEST_VARDIR/tmp/zero_length_file.result
--exec echo "echo ok;" > $MYSQLTEST_VARDIR/tmp/query.sql
--error 1
--exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql -R $MYSQLTEST_VARDIR/tmp/zero_length_file.result 2>&1
--exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql -R $MYSQLTEST_VARDIR/tmp/zero_length_file.result > /dev/null 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.result;
remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.reject;
#
# Test that a test file that does not generate any output fails.
#
......@@ -1407,6 +1419,8 @@ select "this will be executed";
--error 1
--exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql 2>&1
remove_file $MYSQLTEST_VARDIR/tmp/query.sql;
#
# Test that mysqltest fails when there are no queries executed
# but a result file exists
......@@ -1436,6 +1450,7 @@ echo Failing multi statement query;
--exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/bug11731.sql 2>&1
drop table t1;
--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
--error 1
--exec $MYSQL_TEST --record -x $MYSQLTEST_VARDIR/tmp/bug11731.sql -R $MYSQLTEST_VARDIR/tmp/bug11731.out 2>&1
# The .out file should be non existent
......@@ -1462,6 +1477,9 @@ drop table t1;
# The .out file should exist
--exec test -s $MYSQLTEST_VARDIR/tmp/bug11731.out
drop table t1;
remove_file $MYSQLTEST_VARDIR/tmp/bug11731.out;
remove_file $MYSQLTEST_VARDIR/tmp/bug11731.log;
remove_file $MYSQLTEST_VARDIR/tmp/bug11731.sql;
#
# Bug#19890 mysqltest: "query" command is broken
......@@ -1607,19 +1625,48 @@ for diff_file command
of mysqltest
EOF
--write_file $MYSQLTEST_VARDIR/tmp/diff4.tmp
Some data
for diff_file command
of musqltest
EOF
# Compare equal files
--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
--diff_files $MYSQLTEST_VARDIR/tmp/diff2.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
# Compare files that differ
# Write the below commands to a intermediary file and execute them with
# mysqltest in --exec, since the output will vary depending on what "diff"
# is available it is sent to /dev/null
--write_file $MYSQLTEST_VARDIR/tmp/diff.test
# Compare files that differ in size
--error 2
--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff3.tmp
--error 2
--diff_files $MYSQLTEST_VARDIR/tmp/diff3.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
# Compare files that differ only in content
--error 1
--diff_files $MYSQLTEST_VARDIR/tmp/diff3.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff4.tmp
--error 1
--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff3.tmp
--diff_files $MYSQLTEST_VARDIR/tmp/diff4.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
EOF
# Execute the above diffs, and send their output to /dev/null - only
# interesting to see that it returns correct error codes
--exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/diff.test > /dev/null 2>&1
# Compare equal files, again...
--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
--remove_file $MYSQLTEST_VARDIR/tmp/diff1.tmp
--remove_file $MYSQLTEST_VARDIR/tmp/diff2.tmp
--remove_file $MYSQLTEST_VARDIR/tmp/diff3.tmp
--remove_file $MYSQLTEST_VARDIR/tmp/diff4.tmp
--remove_file $MYSQLTEST_VARDIR/tmp/diff.test
# ----------------------------------------------------------------------------
# test for file_exist
# ----------------------------------------------------------------------------
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7