#45 move some mysql tests to github

scripts/run.mysql.tests.bash

+#!/usr/bin/env bash
+# ident 4, no tabs
+
+function usage() {
+    echo "run the tokudb mysql tests"
+    echo "--mysqlbuild=$mysqlbuild"
+    echo "--commit=$commit"
+    echo "--tests=$tests --engine=$engine"
+}
+
+function retry() {
+    local cmd
+    local retries
+    local exitcode
+    cmd=$*
+    let retries=0
+    while [ $retries -le 10 ] ; do
+        echo `date` $cmd
+        bash -c "$cmd"
+        exitcode=$?
+        echo `date` $cmd $exitcode $retries
+        let retries=retries+1
+        if [ $exitcode -eq 0 ] ; then break; fi
+        sleep 10
+    done
+    test $exitcode = 0
+}
+
+svnserver=https://svn.tokutek.com/tokudb
+basedir=$HOME/svn.build
+builddir=$basedir/mysql.build
+mysqlbuild=
+mysql_basedir=/usr/local/mysql
+mysqlserver=`hostname`
+commit=0
+tests="*"
+engine=""
+parallel=auto
+
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [[ $arg =~ --(.*)=(.*) ]] ; then
+        eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+    else
+        usage; exit 1
+    fi
+done
+
+if [[ $mysqlbuild =~ (.*)-(tokudb\-.*)-(linux)-(x86_64) ]] ; then
+    mysql=${BASH_REMATCH[1]}
+    tokudb=${BASH_REMATCH[2]}
+    system=${BASH_REMATCH[3]}
+    arch=${BASH_REMATCH[4]}
+else
+    exit 1
+fi
+
+if [ -d $mysql_basedir/lib/mysql ] ; then
+    export LD_LIBRARY_PATH=$mysql_basedir/lib/mysql
+fi
+
+# update the build directory
+if [ ! -d $basedir ] ; then mkdir $basedir ; fi
+
+pushd $basedir
+if [ $? != 0 ] ; then exit 1; fi
+
+if [ ! -d $builddir ] ; then mkdir $builddir; fi
+
+# make the subversion directory that will hold the test results
+date=`date +%Y%m%d`
+testresultsdir=$builddir/$date
+pushd $builddir
+if [ $? = 0 ] ; then
+    while [ ! -d $date ] ; do
+        svn mkdir $svnserver/mysql.build/$date -m ""
+        svn checkout -q $svnserver/mysql.build/$date
+        if [ $? -ne 0 ] ; then rm -rf $date; fi
+    done
+    popd
+fi
+
+# generate a trace file name
+if [ -z $engine ] ; then
+    tracefile=mysql-test-$mysqlbuild-$mysqlserver
+else
+    tracefile=mysql-engine-$engine-$mysqlbuild-$mysqlserver
+fi
+echo >$testresultsdir/$tracefile
+
+if [ -z $engine ] ; then
+
+    # run all test suites including main
+    teststorun_original="main"
+    teststorun_tokudb=""
+    pushd $mysql_basedir/mysql-test/suite
+    if [ $? = 0 ] ; then
+        for t in $tests ;  do
+            if [[ $t =~ .*\.xfail$ ]] ; then continue; fi
+            if [ $t = "perfschema_stress" ] ; then continue; fi
+            if [ $t = "large_tests" ] ; then continue; fi
+            if [ $t = "pbxt" ] ; then continue; fi
+            if [ -d $t/t ] ; then 
+                if [[ $t =~ tokudb* ]] ; then
+                    if [ -z $teststorun_tokudb ] ; then teststorun_tokudb="$t" ; else teststorun_tokudb="$teststorun_tokudb,$t"; fi
+                else
+                    teststorun_original="$teststorun_original,$t";
+                fi
+            fi
+        done
+        popd
+    fi
+  
+    # run the tests
+    pushd $mysql_basedir/mysql-test
+    if [ $? = 0 ] ; then
+ 
+        if [[ $mysqlbuild =~ mysql-5.6 ]] || [[ $mysqlbuild =~ mariadb-10 ]] || [[ $mysqlbuild =~ Percona ]] ; then
+            ./mysql-test-run.pl --suite=$teststorun_original --big-test --mysqld=--loose-tokudb-debug=3072 --max-test-fail=0 --force --retry=1 --testcase-timeout=60 \
+                --parallel=$parallel >>$testresultsdir/$tracefile 2>&1
+        else
+            ./mysql-test-run.pl --suite=$teststorun_original --big-test --mysqld=--loose-tokudb-debug=3072 --max-test-fail=0 --force --retry=1 --testcase-timeout=60 \
+                --parallel=$parallel --mysqld=--default-storage-engine=myisam --mysqld=--sql-mode="" >>$testresultsdir/$tracefile 2>&1
+        fi
+        
+        if [[ $mysqlbuild =~ Percona ]] ; then
+            ./mysql-test-run.pl --suite=$teststorun_tokudb --big-test --mysqld=--plugin-load=tokudb=ha_tokudb.so --mysqld=--loose-tokudb-debug=3072 --max-test-fail=0 --force --retry=1 --testcase-timeout=60 \
+                --parallel=$parallel >>$testresultsdir/$tracefile 2>&1  
+        else
+            ./mysql-test-run.pl --suite=$teststorun_tokudb --big-test --mysqld=--loose-tokudb-debug=3072 --max-test-fail=0 --force --retry=1 --testcase-timeout=60 \
+                --parallel=$parallel >>$testresultsdir/$tracefile 2>&1  
+        fi
+        exitcode=$?
+        popd
+    fi
+    
+    engine="tokudb"
+fi
+
+if [ ! -z $engine ] ; then
+    teststorun="engines/funcs,engines/iuds"
+    pushd $mysql_basedir/mysql-test
+    if [ $? = 0 ] ; then
+        ./mysql-test-run.pl --suite=$teststorun --mysqld=--default-storage-engine=$engine --force --retry-failure=0 --max-test-fail=0 --nowarnings \
+            --parallel=$parallel >>$testresultsdir/$tracefile 2>&1
+        exitcode=$?
+        popd
+    fi
+fi
+
+# summarize the results
+let tests_failed=0
+let tests_passed=0
+while read line ; do
+    if [[ "$line" =~ (Completed|Timeout):\ Failed\ ([0-9]+)\/([0-9]+) ]] ; then
+            # failed[2]/total[3]
+        let tests_failed=tests_failed+${BASH_REMATCH[2]}
+        let tests_passed=tests_passed+${BASH_REMATCH[3]}-${BASH_REMATCH[2]}
+    elif [[ "$line" =~ Completed:\ All\ ([0-9]+)\ tests ]] ; then
+            # passed[1]
+        let tests_passed=tests_passed+${BASH_REMATCH[1]}
+    fi
+done <$testresultsdir/$tracefile
+
+# commit the results
+if [ $exitcode = 0 -a $tests_failed = 0 ] ; then
+    testresult="PASS=$tests_passed"
+else
+    testresult="FAIL=$tests_failed PASS=$tests_passed"
+fi
+pushd $testresultsdir
+if [ $? = 0 ] ; then
+    if [ $commit != 0 ] ; then
+        svn add $tracefile 
+        if [[ $tracefile =~ "mysql-test" ]] ; then test=mysql-test; else test=mysql-engine-$engine; fi
+        retry svn commit -m \"$testresult $test $mysqlbuild $mysqlserver\" $tracefile 
+    fi
+    popd
+fi
+
+popd # $basedir
+
+if [[ $testresult =~ "PASS" ]] ; then exitcode=0; else exitcode=1; fi
+exit $exitcode
+
+

scripts/run.sql.bench.bash

+#!/usr/bin/env bash
+
+function usage() {
+    echo "run the sql bench tests"
+    echo "--mysqlbuild=$mysqlbuild"
+    echo "--commit=$commit"
+}
+
+function retry() {
+    local cmd
+    local retries
+    local exitcode
+    cmd=$*
+    let retries=0
+    while [ $retries -le 10 ] ; do
+        echo `date` $cmd
+        bash -c "$cmd"
+        exitcode=$?
+        echo `date` $cmd $exitcode $retries
+        let retries=retries+1
+        if [ $exitcode -eq 0 ] ; then break; fi
+        sleep 10
+    done
+    test $exitcode = 0
+}
+
+svnserver=https://svn.tokutek.com/tokudb
+basedir=$HOME/svn.build
+builddir=$basedir/mysql.build
+mysqlbuild=
+mysqlserver=`hostname`
+commit=0
+engine=tokudb
+socket=/tmp/mysql.sock
+system=`uname -s | tr [:upper:] [:lower:]`
+arch=`uname -m | tr [:upper:] [:lower:]`
+
+# parse the command line
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [[ $arg =~ --(.*)=(.*) ]] ; then
+        eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+    else
+        usage; exit 1
+    fi
+done
+
+if [[ $mysqlbuild =~ (.*)-(tokudb-.*)-(linux)-(x86_64) ]] ; then
+    mysql=${BASH_REMATCH[1]}
+    tokudb=${BASH_REMATCH[2]}
+    system=${BASH_REMATCH[3]}
+    arch=${BASH_REMATCH[4]}
+else
+    exit 1
+fi
+
+# goto the base directory
+if [ ! -d $basedir ] ; then mkdir $basedir; fi
+pushd $basedir
+
+# update the build directory
+if [ ! -d $builddir ] ; then mkdir $builddir; fi
+
+date=`date +%Y%m%d`
+testresultsdir=$builddir/$date
+pushd $builddir
+while [ ! -d $date ] ; do
+    svn mkdir $svnserver/mysql.build/$date -m ""
+    svn checkout -q $svnserver/mysql.build/$date
+    if [ $? -ne 0 ] ; then rm -rf $date; fi
+done
+popd
+
+# run the tests
+pushd /usr/local/mysql/sql-bench
+
+tracefile=sql-bench-$engine-$mysqlbuild-$mysqlserver.trace
+summaryfile=sql-bench-$engine-$mysqlbuild-$mysqlserver.summary
+
+function mydate() {
+    date +"%Y%m%d %H:%M:%S"
+}
+
+function runtests() {
+    testargs=$*
+    for testname in test-* ; do
+        chmod +x ./$testname
+        echo `mydate` $testname $testargs
+        ./$testname $testargs
+        exitcode=$?
+        echo `mydate`
+        if [ $exitcode != 0 ] ; then
+            # assume that the test failure due to a crash.  allow mysqld to restart.
+            sleep 60
+        fi
+    done
+}
+
+>$testresultsdir/$tracefile
+
+runtests --create-options=engine=$engine --socket=$socket --verbose --small-test         >>$testresultsdir/$tracefile 2>&1
+runtests --create-options=engine=$engine --socket=$socket --verbose --small-test --fast  >>$testresultsdir/$tracefile 2>&1
+runtests --create-options=engine=$engine --socket=$socket --verbose                      >>$testresultsdir/$tracefile 2>&1
+runtests --create-options=engine=$engine --socket=$socket --verbose              --fast  >>$testresultsdir/$tracefile 2>&1
+runtests --create-options=engine=$engine --socket=$socket --verbose              --fast --fast-insert >>$testresultsdir/$tracefile 2>&1
+runtests --create-options=engine=$engine --socket=$socket --verbose              --fast --lock-tables >>$testresultsdir/$tracefile 2>&1
+
+popd
+
+# summarize the results
+python ~/bin/sql.bench.summary.py <$testresultsdir/$tracefile >$testresultsdir/$summaryfile
+
+testresult=""
+pf=`mktemp`
+egrep "^PASS" $testresultsdir/$summaryfile >$pf 2>&1
+if [ $? -eq 0 ] ; then testresult="PASS=`cat $pf | wc -l` $testresult"; fi
+egrep "^FAIL" $testresultsdir/$summaryfile >$pf 2>&1
+if [ $? -eq 0 ] ; then testresult="FAIL=`cat $pf | wc -l` $testresult"; fi
+rm $pf
+if [ "$testresult" = "" ] ; then testresult="?"; fi
+
+# commit the results
+pushd $testresultsdir
+if [ $commit != 0 ] ; then
+    svn add $tracefile $summaryfile
+    retry svn commit -m \"$testresult sql-bench $mysqlbuild $mysqlserver\" $tracefile $summaryfile
+fi
+popd
+
+popd
+
+if [[ $testresult =~ "PASS" ]] ; then exitcode=0; else exitcode=1; fi
+exit $exitcode
+
+
+

scripts/setup.mysql.bash

+#!/usr/bin/env bash
+
+function usage() {
+    echo "setup.mysql.bash"
+    echo "--mysqlbuild=$mysqlbuild --shutdown=$shutdown --install=$install --startup=$startup"
+}
+
+mysqlbuild=
+shutdown=1
+install=1
+startup=1
+s3bucket=tokutek-mysql-build
+builtins="mysqlbuild shutdown install startup s3bucket"
+mysqld_args="--user=mysql --core-file --core-file-size=unlimited"
+defaultsfile=""
+if [ -f /etc/$(whoami).my.cnf ] ; then
+    defaultsfile=/etc/$(whoami).my.cnf
+fi
+
+function is_builtin() {
+    local v=$1; shift
+    local x
+    for x in $* ; do 
+        if [ $v = $x ] ; then echo 1; return; fi
+    done
+    echo 0
+}
+
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [ $arg = "--help" -o $arg = "-h" -o $arg = "-?" ] ; then
+        usage; exit 1
+    elif [[ $arg =~ --(.*)=(.*) ]] ; then
+        r=$(is_builtin ${BASH_REMATCH[1]} $builtins)
+        if [ $r = 1 ] ; then
+            eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+        else
+            mysqld_args="$mysqld_args $arg"
+        fi
+    else
+        mysqld_args="$mysqld_args $arg"
+    fi
+done
+
+if [[ $mysqlbuild =~ (.*)-(tokudb\-.*)-(linux)-(x86_64) ]] ; then
+    mysql=${BASH_REMATCH[1]}
+    tokudb=${BASH_REMATCH[2]}
+    system=${BASH_REMATCH[3]}
+    arch=${BASH_REMATCH[4]}
+else
+    exit 1
+fi
+mysqltarball=$mysqlbuild.tar.gz
+
+if [ ! -d downloads ] ; then mkdir downloads; fi
+
+pushd downloads
+if [ $? != 0 ] ; then exit 1; fi
+
+basedir=$PWD
+
+# get the release
+if [ ! -f $mysqltarball ] ; then
+    s3get $s3bucket $mysqltarball $mysqltarball 
+    if [ $? -ne 0 ] ; then exit 1; fi
+fi
+if [ ! -f $mysqltarball.md5 ] ; then
+    s3get $s3bucket $mysqltarball.md5 $mysqltarball.md5
+    if [ $? -ne 0 ] ; then exit 1; fi
+fi
+
+md5sum --check $mysqltarball.md5
+if [ $? -ne 0 ] ; then exit 1; fi
+
+# shutdown mysql
+if [ $shutdown -ne 0 ] ; then
+    if [ -x /etc/init.d/mysql ] ; then
+        sudo setsid /etc/init.d/mysql stop
+    else
+        /usr/local/mysql/bin/mysqladmin shutdown
+    fi
+    sleep 60
+fi
+
+pushd /usr/local
+if [ $? = 0 ] ; then 
+    rm mysql
+    popd
+fi
+
+# install the release
+pushd /usr/local/mysqls 2>/dev/null
+if [ $? = 0 ] ; then
+    mysqldir=mysqls/$mysqlbuild
+else
+    pushd /usr/local
+    if [ $? -ne 0 ] ; then exit 1; fi
+    mysqldir=$mysqlbuild
+fi
+
+if [ ! -d $mysqlbuild ] || [ $install -ne 0 ] ; then
+    rm mysql
+    if [ -d $mysqlbuild ] ; then sudo rm -rf $mysqlbuild; fi
+
+    tar xzf $basedir/$mysqltarball
+    if [ $? -ne 0 ] ; then exit 1; fi
+    ln -s $mysqldir /usr/local/mysql
+
+    installdb=$mysqlbuild/bin/mysql_install_db
+    if [ ! -f $installdb ] ; then
+        installdb=$mysqlbuild/scripts/mysql_install_db
+    fi
+
+    sudo chown -R mysql $mysqlbuild/data
+    sudo chgrp -R mysql $mysqlbuild/data
+
+    # 5.6 debug build needs this 
+    if [ ! -f $mysqlbuild/bin/mysqld ] && [ -f $mysqlbuild/bin/mysqld-debug ] ; then
+	ln $mysqlbuild/bin/mysqld-debug $mysqlbuild/bin/mysqld
+    fi
+
+    if [ -z "$defaultsfile" ] ; then
+        sudo $installdb --user=mysql --basedir=$PWD/$mysqlbuild --datadir=$PWD/$mysqlbuild/data
+    else
+        sudo $installdb --defaults-file=$defaultsfile --user=mysql --basedir=$PWD/$mysqlbuild --datadir=$PWD/$mysqlbuild/data
+    fi
+    if [ $? -ne 0 ] ; then exit 1; fi
+    
+else
+    # create link
+    ln -s $mysqldir /usr/local/mysql
+    if [ $? -ne 0 ] ; then exit 1; fi
+fi
+popd
+
+# start mysql
+if [ $startup -ne 0 ] ; then 
+    ulimit -a
+    # increase the open file limit
+    ulimit -n 10240
+    exitcode=$?
+    echo ulimit -n 10240 exitcode $exitcode
+
+    if [ -x /etc/init.d/mysql ] ; then
+        sudo setsid /etc/init.d/mysql start
+    else
+        sudo -b /usr/local/mysql/bin/mysqld_safe $mysqld_args >/dev/null 2>&1 &
+    fi
+    sleep 60
+
+    # add mysql grants
+    /usr/local/mysql/bin/mysql -u root -e "grant all on *.* to tokubuild@localhost"
+    /usr/local/mysql/bin/mysql -u root -e "grant all on *.* to 'ec2-user'@localhost"
+fi
+
+popd
+
+exit 0

scripts/sql.bench.summary.py

+#!/usr/bin/env python
+
+# summarize the sql-bench trace file
+
+import sys
+import re
+import os.path
+
+class testreports:
+    def __init__(self):
+        self.reports = []
+
+    def append(self, report):
+        self.reports.append(report)
+
+    def duration(self, start, stop):
+        t0 = os.popen('date -d"' + start + '" +%s').readline()
+        t1 = os.popen('date -d"' + stop + '" +%s').readline()
+        return int(t1) - int(t0)
+
+    def printit(self, i):
+        report = self.reports[i]
+        d = self.duration(report["start"], report["stop"])
+        print "%s %s %6u %s" % (report["result"].upper(), report["start"], d, report["name"])
+        # print self.reports[i]
+
+    def printall(self):
+        for i in range(len(self.reports)):
+            self.printit(i)
+
+    def stoptime(self, stoptime):
+        if len(self.reports) > 0:
+            lastreport = self.reports[-1]
+            lastreport["stop"] = stoptime
+
+def main():
+    reports = testreports()
+    testreport = {}
+    while 1:
+        b = sys.stdin.readline()
+        if b == "": break
+        
+        b = b.rstrip('\n')
+
+        match = re.match("^(\d{8} \d{2}:\d{2}:\d{2})$", b)
+        if match:
+            if totaltime == "" and testreport["result"] == "pass":
+                testreport["result"] = "fail"
+            testreport["stop"] = match.group(1)
+            reports.append(testreport)
+            testreport = {}
+            continue
+        
+        match = re.match("^(\d{8} \d{2}:\d{2}:\d{2}) (test-.*)$", b)
+        if match:
+            testreport["start"] = match.group(1)
+            testreport["name"] = match.group(2)
+            testreport["result"] = "pass"
+            totaltime = ""
+            continue
+        
+        match = re.match(".*Got error|.*Died at", b)
+        if match: testreport["result"] = "fail"
+
+        match = re.match("^Total time|^Estimated total time", b)
+        if match: totaltime = b
+
+        match = re.match("skip", b)
+        if match: testreport["result"] = "skip"
+                
+    reports.printall()
+        
+    return 0
+
+sys.exit(main())