Merge branch 'master' of github.com:Tokutek/ft-engine

scripts/common.sh

@@ -161,7 +161,8 @@ function parse_mysqlbuild() {
             if [ -z $jemalloc_tree ] ; then jemalloc_tree=$jemalloc_version; fi
         fi
         # 5.6 is temp in  another repo
-        if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
+        mysql_repo=$mysql_distro
+        if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_repo=mysql56; fi
     else
         exitcode=1
     fi
@@ -174,7 +175,8 @@ function parse_mysql() {
     if [[ $mysql =~ ^(mysql|mariadb)-(.*)$ ]] ; then
         mysql_distro=${BASH_REMATCH[1]}
         mysql_version=${BASH_REMATCH[2]}
-        if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
+        mysql_repo=$mysql_distro
+        if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_repo=mysql56; fi
         exitcode=0
     else
         exitcode=1

scripts/make.mysql.bash

@@ -97,7 +97,7 @@ fi
 
 # download all the mysql source
 if [ ! -d $mysql_distro ] ; then
-    github_download Tokutek/$mysql_distro $(git_tree $git_tag $mysql_tree) $mysql_distro
+    github_download Tokutek/$mysql_repo $(git_tree $git_tag $mysql_tree) $mysql_distro
 fi
 
 cd $mysql_distro

scripts/run.atc.ontime.bash

@@ -7,7 +7,7 @@ function usage() {
     echo "[--dbname=$dbname]"
     echo "[--load=$load] [--check=$check] [--run=$run]"
     echo "[--engine=$engine]"
-    echo "[--tokudb_load_save_space=$tokudb_load_save_space]"
+    echo "[--tokudb_load_save_space=$tokudb_load_save_space] [--tokudb_row_format=$tokudb_row_format] [--tokudb_loader_memory_size=$tokudb_loader_memory_size]"
 }
 
 function retry() {
@@ -43,6 +43,8 @@ check=1
 run=1
 engine=tokudb
 tokudb_load_save_space=0
+tokudb_row_format=
+tokudb_loader_memory_size=
 verbose=0
 svn_server=https://svn.tokutek.com/tokudb
 svn_branch=.
@@ -97,6 +99,8 @@ if [ $dbname = "atc" -a $engine != "tokudb" ] ; then dbname="atc_$engine"; fi
 
 runfile=$testresultsdir/$dbname-$tblname-$mysqlbuild-$mysqlserver
 if [ $tokudb_load_save_space != 0 ] ; then runfile=$runfile-compress; fi
+if [ "$tokudb_row_format" != "" ] ; then runfile=$runfile-$tokudb_row_format; fi
+if [ "$tokudb_loader_memory_size" != "" ] ; then runfile=$runfile-$tokudb_loader_memory_size; fi
 rm -rf $runfile
 
 testresult="PASS"
@@ -162,6 +166,14 @@ if [ $load -ne 0 -a $testresult = "PASS" ] ; then
     if [ $exitcode -ne 0 ] ; then testresult="FAIL"; fi
 fi
 
+if [ $load -ne 0 -a $testresult = "PASS" -a "$tokudb_row_format" != "" ] ; then
+    echo `date` create table $dbname.$tblname >>$runfile
+    mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "alter table $tblname row_format=$tokudb_row_format"  >>$runfile 2>&1
+    exitcode=$?
+    echo `date` create table $exitcode >>$runfile
+    if [ $exitcode -ne 0 ] ; then testresult="FAIL"; fi
+fi
+
 if [ $load -ne 0 -a $testresult = "PASS" -a $engine != "tokudb" ] ; then
     echo `date` alter table $engine >>$runfile
     mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "alter table $tblname engine=$engine"  >>$runfile 2>&1
@@ -170,6 +182,8 @@ if [ $load -ne 0 -a $testresult = "PASS" -a $engine != "tokudb" ] ; then
     if [ $exitcode -ne 0 ] ; then testresult="FAIL"; fi
 fi  
 
+mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "show create table $tblname"  >>$runfile 2>&1
+
 # load the data
 if [ $load -ne 0 -a $testresult = "PASS" ] ; then
     echo `date` load data >>$runfile

scripts/run.tpch.bash

@@ -5,7 +5,8 @@ function usage() {
     echo "[--SCALE=$SCALE] [--ENGINE=$ENGINE]"
     echo "[--dbgen=$dbgen] [--load=$load] [--check=$check] [--compare=$compare] [--query=$query]"
     echo "[--mysqlbuild=$mysqlbuild] [--commit=$commit]"
-    echo "[--testinstance=$testinstance] [--tokudb_load_save_space=$tokudb_load_save_space]"
+    echo "[--testinstance=$testinstance]"
+    echo "[--tokudb_load_save_space=$tokudb_load_save_space] [--tokudb_row_format=$tokudb_row_format] [--tokudb_loader_memory_size=$tokudb_loader_memory_size]"
 }
 
 function retry() {
@@ -46,6 +47,8 @@ system=`uname -s | tr [:upper:] [:lower:]`
 arch=`uname -m | tr [:upper:] [:lower:]`
 testinstance=
 tokudb_load_save_space=0
+tokudb_row_format=
+tokudb_loader_memory_size=
 svn_server=https://svn.tokutek.com/tokudb
 svn_branch=.
 svn_revision=HEAD
@@ -105,6 +108,8 @@ fi
 
 runfile=$testresultsdir/$dbname
 if [ $tokudb_load_save_space != 0 ] ; then runfile=$runfile-compress; fi
+if [ "$tokudb_row_format" != "" ] ; then runfile=$runfile-$tokudb_row_format; fi
+if [ "$tokudb_loader_memory_size" != "" ] ; then runfile=$runfile-$tokudb_loader_memory_size; fi
 runfile=$runfile-$mysqlbuild-$mysqlserver
 rm -rf $runfile
 
@@ -210,13 +215,25 @@ if [ $load != 0 -a $testresult = "PASS" ] ; then
     if [ $exitcode -ne 0 ] ; then testresult="FAIL"; fi
 fi
 
+# get the current loader memory size
+if [ $load != 0 -a $testresult = "PASS" ] ; then
+    let default_loader_memory_size="$(mysql -S $mysqlsocket -u $mysqluser -e'select @@tokudb_loader_memory_size' --silent --skip-column-names)"
+    exitcode=$?
+    echo `date` get tokudb_loader_memory_size $exitcode >>$runfile
+    if [ $exitcode -ne 0 ] ; then testresult="FAIL"; fi
+    if [ "$tokudb_loader_memory_size" = "" ] ; then tokudb_loader_memory_size=$default_loader_memory_size; fi
+fi
+
 # load the data
 if [ $load != 0 -a $testresult = "PASS" ] ; then
     for tblname in $TABLES ; do
         echo `date` load table $tblname >>$runfile
         ls -l $tpchdir/data/tpch${SCALE}G/$tblname.tbl >>$runfile
         start=$(date +%s)
-        mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "set session tokudb_load_save_space=$tokudb_load_save_space; load data infile '$tpchdir/data/tpch${SCALE}G/$tblname.tbl' into table $tblname fields terminated by '|'" >>$runfile 2>&1
+        mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "set global tokudb_loader_memory_size=$tokudb_loader_memory_size;\
+            set tokudb_load_save_space=$tokudb_load_save_space;\
+            load data infile '$tpchdir/data/tpch${SCALE}G/$tblname.tbl' into table $tblname fields terminated by '|';\
+            set global tokudb_loader_memory_size=$default_loader_memory_size" >>$runfile 2>&1
         exitcode=$?
         let loadtime=$(date +%s)-$start
         echo `date` load table $tblname $exitcode loadtime=$loadtime>>$runfile
@@ -228,7 +245,11 @@ if [ $check != 0 -a $testresult = "PASS" ] ; then
     for tblname in lineitem ; do
         echo `date` add clustering index $tblname >>$runfile
         start=$(date +%s)
-        mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "set session tokudb_create_index_online=0;create clustering index i_shipdate on lineitem (l_shipdate)" >>$runfile 2>&1
+        mysql -S $mysqlsocket -u $mysqluser -D $dbname -e "set global tokudb_loader_memory_size=$tokudb_loader_memory_size;\
+            set tokudb_load_save_space=$tokudb_load_save_space;\
+            set session tokudb_create_index_online=0;\
+            create clustering index i_shipdate on lineitem (l_shipdate);\
+            set global tokudb_loader_memory_size=$default_loader_memory_size" >>$runfile 2>&1
         exitcode=$?
         let loadtime=$(date +%s)-$start
         echo `date` add clustering index $tblname $exitcode loadtime=$loadtime >>$runfile