Skip to content

M
mariadb
Commit aa59bca6 authored by Leif Walsh's avatar Leif Walsh
Browse files
Options

rename make.mysql.new.bash so it is the new default, closes #10

parent 805a0c7d
Show whitespace changes
Inline Side-by-side
Showing with 870 additions and 870 deletions
scripts/make.mysql.bash
View file @ aa59bca6
#!/usr/bin/env bash
#!/bin/bash
# build mysql with the tokudb storage engine and the fractal tree
#
# parse arguments
# checkout the fractal tree
# checkout jemalloc
# build the fractal tree
# checkout the mysql source
# checkout the tokudb storage engine
# checkout the tokudb backup source
# generate a mysql build script
# generate a mysql source tarball
# execute the build script
# copy build files to an amazon s3 bucket
shopt -s compat31 2> /dev/null
set -e
set -u
function usage() {
echo "make.mysql.bash - build mysql with the tokudb storage engine and the fractal tree"
echo "make mysql with the tokudb storage engine from github repo's"
echo "--git_tag=$git_tag"
echo "--mysqlbuild=$mysqlbuild"
echo "--mysql=$mysql"
echo "--tokudb_version=$tokudb_version"
echo "--mysql=$mysql --mysql_distro=$mysql_distro --mysql_version=$mysql_version"
echo "--build_type=$build_type"
echo "--build_debug=$build_debug"
echo "--build_tgz=$build_tgz"
echo "--build_rpm=$build_rpm"
echo "--cc=$cc --cxx=$cxx"
echo "--do_make_check=$do_make_check"
echo "--do_s3=$do_s3 --s3_build_bucket=$s3_build_bucket --s3_release_bucket=$s3_release_bucket"
echo
echo "community release builds using the tokudb-7.0.1 git tag"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.0.1-linux-x86_64"
......@@ -46,618 +26,9 @@ function usage() {
return 1
}
function retry() {
local cmd
local retries
local exitcode
cmd=$*
let retries=0
while [ $retries -le 10 ] ; do
echo `date` $cmd
bash -c "$cmd"
exitcode=$?
echo `date` $cmd $exitcode $retries
let retries=retries+1
if [ $exitcode -eq 0 ] ; then break; fi
sleep 10
done
test $exitcode = 0
}
function github_download() {
repo=$1; shift
rev=$1; shift
dest=$1; shift
mkdir $dest
if [ ! -z $github_token ] ; then
retry curl \
--header "Authorization:\\ token\\ $github_token" \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
if [ $? != 0 ] ; then return; fi
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
elif [ ! -z $github_user ] ; then
retry curl \
--user $github_user \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
if [ $? != 0 ] ; then return; fi
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
elif [ $github_use_ssh != 0 ] ; then
tempdir=$(mktemp -d -p $PWD)
retry git clone git@github.com:${repo}.git $tempdir
if [ $? != 0 ] ; then return; fi
pushd $tempdir
if [ $? != 0 ] ; then return; fi
git checkout $rev
if [ $? != 0 ] ; then return; fi
popd
# export the right branch or tag
(cd $tempdir ;
git archive \
--format=tar \
$rev) | \
tar --extract \
--directory $dest
if [ $? != 0 ] ; then return; fi
rm -rf $tempdir
else
retry curl \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
fi
}
# returns b if b is defined else returns a
function git_tree() {
local a=$1; shift
local b=$1; shift
if [ ! -z $b ] ; then
echo $b
else
echo $a;
fi
}
# compute the number of cpus in this system. used to parallelize the build.
function get_ncpus() {
if [ -f /proc/cpuinfo ]; then
grep bogomips /proc/cpuinfo | wc -l
elif [ $system = darwin ] ; then
sysctl -n hw.ncpu
else
echo 1
fi
}
# check out the fractal tree source from subversion, build it, and make the fractal tree tarballs
function build_fractal_tree() {
if [ ! -d ft-index ] ; then
# get the ft-index repo
github_download Tokutek/ft-index $(git_tree $git_tag $ftindex_tree) ft-index
if [ $? != 0 ] ; then exit 1; fi
# get jemalloc
github_download Tokutek/jemalloc $(git_tree $git_tag $jemalloc_tree) ft-index/third_party/jemalloc
if [ $? != 0 ] ; then exit 1; fi
# get the commit id of the ft-index repo
local ft_revision=0
if [ ! -z $github_token ] ; then
local ft_index_ls
ft_index_ls=$(git ls-remote https://$github_token:x-oauth-basic@github.com/Tokutek/ft-index.git $git_tag)
if [ $? != 0 ] ; then exit 1; fi
local ft_revision_hex
ft_revision_hex=0x$(echo $ft_index_ls | cut -c-7)
let ft_revision=$ft_revision_hex
fi
pushd ft-index
if [ $? != 0 ] ; then exit 1; fi
local ft_build_type=""
local use_valgrind=""
local debug_paranoid=""
if [[ $build_debug = 1 ]]; then
ft_build_type=Debug
use_valgrind="ON"
debug_paranoid="ON"
else
ft_build_type=Release
use_valgrind="OFF"
debug_paranoid="OFF"
fi
mkdir -p build
pushd build
if [ $? != 0 ] ; then exit 1; fi
#-D JEMALLOC_SOURCE_DIR=../../jemalloc \
CC=$cc CXX=$cxx cmake \
-D LIBTOKUDB=$tokufractaltree \
-D LIBTOKUPORTABILITY=$tokuportability \
-D CMAKE_TOKUDB_REVISION=$ft_revision \
-D CMAKE_BUILD_TYPE=$ft_build_type \
-D CMAKE_INSTALL_PREFIX=../../$tokufractaltreedir \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
-D USE_CTAGS=OFF \
-D USE_ETAGS=OFF \
-D USE_CSCOPE=OFF \
-D USE_VALGRIND=$use_valgrind \
-D TOKU_DEBUG_PARANOID=$debug_paranoid \
..
make install -j$makejobs
if [ $? != 0 ] ; then exit 1; fi
popd # build
popd # ft-index
pushd $tokufractaltreedir/examples
if [ $? != 0 ] ; then exit 1; fi
# test the examples
sed -ie "s/LIBTOKUDB = tokudb/LIBTOKUDB = $tokufractaltree/" Makefile
sed -ie "s/LIBTOKUPORTABILITY = tokuportability/LIBTOKUPORTABILITY = $tokuportability/" Makefile
if [ $system = darwin ] ; then
DYLD_LIBRARY_PATH=$PWD/../lib:$DYLD_LIBRARY_PATH make check CC=$cc
exitcode=$?
else
if [ $do_make_check != 0 ] ; then
make check CC=$cc
exitcode=$?
else
exitcode=0
fi
fi
echo `date` make check examples $tokufractaltree $exitcode
if [ $exitcode != 0 ] ; then exit 1; fi
make clean
popd
# make tarballs
tar czf $tokufractaltreedir.tar.gz $tokufractaltreedir
md5sum $tokufractaltreedir.tar.gz >$tokufractaltreedir.tar.gz.md5
md5sum --check $tokufractaltreedir.tar.gz.md5
fi
}
function generate_cmake_cmd() {
local mysqlsrc=$1; shift
local extra_flags=$*
if [[ $mysqlsrc =~ ^Percona ]] ; then extra_flags=-DWITH_EMBEDDED_SERVER=OFF; fi
echo -n cmake .. $extra_flags -DBUILD_CONFIG=mysql_release -DCMAKE_BUILD_TYPE=$cmake_build_type
if [ $system = linux ] ; then
echo -n " " \
-DTOKU_JEMALLOC_LIBRARY=\"-Wl,--whole-archive \$TOKUFRACTALTREE/lib/libjemalloc.a -Wl,-no-whole-archive\"
elif [ $system = darwin ] ; then
echo -n " " \
-DWITH_SAFEMALLOC=OFF -DWITH_SSL=system \
-DTOKU_JEMALLOC_LIBRARY=\"-Wl,-force_load -Wl,\$TOKUFRACTALTREE/lib/libjemalloc.a\"
else
exit 1
fi
if [ $build_type = enterprise ] ; then
echo -n " " -DCOMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
echo
}
# generate a script that builds from the mysql src tarball and the fractal tree tarball
function generate_build_from_src() {
local mysqlsrc=$1; local tokufractaltreedir=$2
pushd $mysqlsrc >/dev/null 2>&1
if [ $? = 0 ] ; then
echo '#/usr/bin/env bash'
# defaults
echo 'makejobs=1'
echo 'cc=gcc47'
echo 'cxx=g++47'
# parse arguments
echo 'while [ $# -gt 0 ] ; do'
echo ' arg=$1; shift'
echo ' if [[ $arg =~ --(.*)=(.*) ]] ; then'
echo ' eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}'
echo ' else'
echo ' break'
echo ' fi'
echo 'done'
# check tarballs
echo md5sum --check $mysqlsrc.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo md5sum --check $tokufractaltreedir.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo "if [ ! -d $mysqlsrc ] ; then"
echo " tar xzf $mysqlsrc.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
echo "if [ ! -d $tokufractaltreedir ] ;then"
echo " tar xzf $tokufractaltreedir.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
# setup environment variables
echo export TOKUDB_VERSION=$tokudb_version
echo export TOKUFRACTALTREE='$PWD'/$tokufractaltreedir
echo export TOKUDB_PATCHES=$tokudb_patches
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}_static
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}_static
echo 'export CC=$cc; export CXX=$cxx'
echo "pushd $mysqlsrc"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [ ! -f CMakeLists.txt ] ; then exit 1; fi
if [ $build_tgz != 0 ] ; then
echo "mkdir build.$cmake_build_type; pushd build.$cmake_build_type"
echo 'if [ $? != 0 ] ; then exit 1; fi'
cmd=$(generate_cmake_cmd $mysqlsrc)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# fixup tarball name mismatch (if any)
echo "if [ ! -f $mysqldir.tar.gz ] ; then"
echo ' oldtb=$(ls *.gz)'
echo ' if [[ $oldtb =~ (.*).tar.gz ]] ; then oldmysqldir=${BASH_REMATCH[1]}; fi'
echo ' tar xzf $oldtb'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo ' mv $oldmysqldir' $mysqldir
echo " tar czf $mysqldir.tar.gz $mysqldir"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
# add ft files to the mysql tarball
echo tar xzf $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add ftdump to mysql tarball
echo "cp ../../$tokufractaltreedir/bin/ftdump $mysqldir/bin/tokuftdump"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# cleanup
echo rm $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo tar czf $mysqldir.tar.gz $mysqldir
echo 'if [ $? != 0 ] ; then exit 1; fi'
# generate the md5 file
echo "if [ -f $mysqldir.tar.gz ] ; then"
echo " md5sum $mysqldir.tar.gz >$mysqldir.tar.gz.md5"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
echo popd
fi
# make RPMs
if [ $system = linux -a $build_rpm != 0 ]; then
echo "mkdir build.${cmake_build_type}.rpms; pushd build.${cmake_build_type}.rpms"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [[ $mysqlsrc =~ ^mariadb ]] ; then
echo linux_distro=linux
echo 'if [ -f /etc/issue ] ; then'
echo ' if [[ "$(head -1 /etc/issue)" =~ Red\ Hat\ Enterprise\ Linux\ Server\ release\ ([56]) ]] ; then linux_distro=rhel${BASH_REMATCH[1]}; fi'
echo ' if [[ "$(head -1 /etc/issue)" =~ CentOS\ release\ ([56]) ]] ; then linux_distro=centos${BASH_REMATCH[1]}; fi'
echo fi
cmd=$(generate_cmake_cmd $mysqlsrc -DRPM=\$linux_distro)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
else
echo 'mkdir -p {tmp,BUILD,RPMS,SOURCES,SPECS,SRPMS}'
# split myslqsrc into components
echo "if [[ $mysqlsrc =~ (.*)-([0-9]+\.[0-9]+\.[0-9]+.*)-tokudb-(.*) ]] ; then"
echo ' mysql_distro=${BASH_REMATCH[1]}'
echo ' mysql_version=${BASH_REMATCH[2]}'
echo ' mysql_distro_version=${mysql_distro}-${mysql_version}'
echo ' tokudb_distro=tokudb'
echo ' tokudb_version=${BASH_REMATCH[3]}'
echo ' tokudb_distro_version=${tokudb_distro}-${tokudb_version}'
echo else
echo ' exit 1'
echo fi
# copy the source tarball
echo "cp ../../$mysqlsrc.tar.gz SOURCES/$mysqlsrc.tar.gz"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# hack on the spec file
echo 'specfile=${mysql_distro}.${mysql_version}.spec'
echo 'cp ../support-files/$specfile SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'sed -i -e"s/^%define mysql_version.*/&-${tokudb_distro_version}/" SPECS/$specfile'
echo 'sed -i -e"s/^%define release.*/%define release $(echo ${tokudb_distro_version}|tr - .)/" SPECS/$specfile'
# echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DEXTRA_VERSION=-${tokudb_distro_version}/" SPECS/$specfile'
# include README-TOKUDB in the RPM
echo 'sed -i -e"s/README/& %{src_dir}\/README-TOKUDB/" SPECS/$specfile'
# add jemalloc to the linker flags
echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DTOKU_JEMALLOC_LIBRARY=\"-Wl,--whole-archive \$\{TOKUFRACTALTREE\}\/lib\/libjemalloc.a -Wl,-no-whole-archive\"/" SPECS/$specfile'
# add the hot backup lib to the server package file list
echo 'sed -i -e"s/%{_datadir}\/mysql\/$/&\n%attr(755, root, root) %{_libdir}\/libHotBackup*.so/" SPECS/$specfile'
# build generic linux RPMs
echo 'export MYSQL_BUILD_MAKE_JFLAG=-j$makejobs'
echo 'rpmbuild -v --define="_topdir $PWD" --define="_tmppath $PWD/tmp" --define="src_base $mysql_distro" -ba SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# move the rpms
echo 'pushd RPMS/$(uname -m)'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ../..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
echo 'pushd SRPMS'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
fi
# generate the md5 file
echo 'rpmfiles=$(ls *.rpm)'
echo 'for x in $rpmfiles; do'
echo ' md5sum $x >$x.md5'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'done'
echo popd
fi
echo popd
popd >/dev/null 2>&1
fi
}
# checkout the mysql source, generate a build script, and make the mysql source tarball
function build_mysql_src() {
if [ ! -d $mysqlsrc ] ; then
# get the mysql repo
if [ ! -d $mysql_distro ] ; then
github_download Tokutek/$mysql_distro $(git_tree $git_tag $mysql_tree) $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
fi
# get the hot backup source
if [ ! -d backup-$build_type ] ; then
github_download Tokutek/backup-$build_type $(git_tree $git_tag $backup_tree) backup-$build_type
if [ $? != 0 ] ; then exit 1; fi
fi
# get the ft-engine repo
if [ ! -d ft-engine ] ; then
github_download Tokutek/ft-engine $(git_tree $git_tag $ftengine_tree) ft-engine
if [ $? != 0 ] ; then exit 1; fi
fi
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
sed -i "" -e"s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${tokudb_version}/" $mysqlsrc/VERSION
# add the backup source
mkdir $mysqlsrc/toku_backup
pushd backup-$build_type/backup
if [ $? != 0 ] ; then exit 1; fi
copy_files=$(ls CMakeLists.txt *.h *.cc *.cmake export.map)
cp $copy_files ../../$mysqlsrc/toku_backup
if [ $? != 0 ] ; then exit 1; fi
popd
# add the tokudb storage engine source
mkdir -p $mysqlsrc/storage/tokudb
pushd ft-engine/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
copy_files=$(ls CMakeLists.txt *.h *.cc)
cp $copy_files ../../../$mysqlsrc/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
popd
# merge the common mysql tests into the source
mv $mysqlsrc/mysql-test $mysqlsrc/mysql-test-save
cp -r ft-engine/mysql-test $mysqlsrc
cp -r $mysqlsrc/mysql-test-save/* $mysqlsrc/mysql-test
rm -rf $mysqlsrc/mysql-test-save
# add the common scripts into the source
cp -r ft-engine/scripts $mysqlsrc
# generate the tokudb.build.bash script from the mysql src and the fractal tree tarballs
generate_build_from_src $mysqlsrc $tokufractaltreedir >$mysqlsrc/scripts/tokudb.build.bash
# add README-TOKUDB
if [ ! -f $mysqlsrc/README-TOKUDB ] ; then
cp ft-index/README-TOKUDB $mysqlsrc
fi
sed -i -e's/FILES README DESTINATION/FILES README README-TOKUDB DESTINATION/' $mysqlsrc/CMakeLists.txt
# make the mysql src tarball
tar czf $mysqlsrc.tar.gz $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
md5sum $mysqlsrc.tar.gz >$mysqlsrc.tar.gz.md5
if [ $? != 0 ] ; then exit 1; fi
fi
}
# make the mysql release tarball
# by executing the build script in the mysql source tarball and compiling with the fractal tree tarball
function build_mysql_release() {
if [ ! -d $mysqlbuild ] ; then
mkdir $mysqlbuild
pushd $mysqlbuild
# create links to the mysql source and fractal tree tarballs
ln $basedir/$mysqlsrc.tar.gz
ln $basedir/$mysqlsrc.tar.gz.md5
ln $basedir/$tokufractaltreedir.tar.gz
ln $basedir/$tokufractaltreedir.tar.gz.md5
# extract the build script
tar xzf $mysqlsrc.tar.gz $mysqlsrc/scripts/tokudb.build.bash
if [ $? != 0 ] ; then exit 1; fi
cp $mysqlsrc/scripts/tokudb.build.bash .
if [ $? != 0 ] ; then exit 1; fi
rm -rf $mysqlsrc
# execute the build script
bash -x tokudb.build.bash --makejobs=$makejobs --cc=$cc --cxx=$cxx
if [ $? != 0 ] ; then exit 1; fi
# move the build files
pushd $mysqlsrc/build.${cmake_build_type}
if [ $? = 0 ] ; then
cp $mysqldir.tar.gz* $basedir; if [ $? != 0 ] ; then exit 1; fi
popd
fi
if [ $build_rpm != 0 ] ; then
pushd $mysqlsrc/build.${cmake_build_type}.rpms
if [ $? = 0 ] ; then
rpmfiles=$(ls *.rpm *.rpm.md5)
for x in $rpmfiles; do cp $x $basedir; if [ $? != 0 ] ; then exit 1; fi; done
popd
fi
fi
popd
fi
}
# split mysql into mysql_distro and mysql_version
function parse_mysql() {
local mysql=$1
if [[ $mysql =~ ^(mysql|mariadb)-(.*)$ ]] ; then
mysql_distro=${BASH_REMATCH[1]}
mysql_version=${BASH_REMATCH[2]}
if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
exitcode=0
else
exitcode=1
fi
test $exitcode = 0
}
# parse a mysqlbuild string and extract the mysql_distro, mysql_version, tokudb_distro, tokudb_version, target_system, and target_arch
# compute build_type, build_debug, and git_tag
function parse_mysqlbuild() {
local mysqlbuild=$1
local exitcode=0
if [[ $mysqlbuild =~ ((mysql|mariadb|percona\-server)-(.*))-((tokudb)-(.*))-(linux|darwin)-(x86_64|i386) ]] ; then
# extract distros and versions from the components
mysql=${BASH_REMATCH[1]}
mysql_distro=${BASH_REMATCH[2]}
mysql_version=${BASH_REMATCH[3]}
tokudb=${BASH_REMATCH[4]}
tokudb_distro=${BASH_REMATCH[5]}
tokudb_version=${BASH_REMATCH[6]}
target_system=${BASH_REMATCH[7]}
target_arch=${BASH_REMATCH[8]}
# verify targets
if [ $target_system != $system ] ; then exitcode=1; fi
if [ $target_arch != $arch ] ; then exitcode=1; fi
local temp_tokudb_version=$tokudb_version
# decode enterprise
if [[ $temp_tokudb_version =~ (.*)-e$ ]] ; then
build_type=enterprise
temp_tokudb_version=${BASH_REMATCH[1]}
else
build_type=community
fi
# decode debug
if [[ $temp_tokudb_version =~ (.*)-debug$ ]] ; then
build_debug=1
temp_tokudb_version=${BASH_REMATCH[1]}
cmake_build_type=Debug
else
build_debug=0
fi
# set tag or HEAD
if [[ $temp_tokudb_version =~ ^([0-9]+)\\.([0-9]+)\\.([0-9]+)$ ]] ; then
git_tag=tokudb-$temp_tokudb_version
else
git_tag=HEAD
# setup _tree defaults
if [ -z $mysql_tree ] ; then mysql_tree=$mysql_version; fi
if [ -z $jemalloc_tree ] ; then jemalloc_tree=$jemalloc_version; fi
fi
# 5.6 is temp in another repo
if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
else
exitcode=1
fi
test $exitcode = 0
}
# copy build files to amazon s3
function copy_to_s3() {
local $s3_build_bucket=$1; local $s3_release_bucket=$2
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3put $s3_build_bucket $f
s3put $s3_build_bucket $f $f
exitcode=$?
# index the file by date
echo `date` s3put $s3_build_bucket $f $exitcode
d=$(date +%Y%m%d)
s3put $s3_build_bucket-date $d/$f /dev/null
exitcode=$?
echo `date` s3put $s3_build_bucket-date $d/$f $exitcode
done
if [[ $git_tag =~ tokudb-.* ]] ; then
s3mkbucket $s3_release_bucket-$git_tag
if [ $? = 0 ] ; then
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3copykey $s3_release_bucket-$git_tag $f
s3copykey $s3_release_bucket-$git_tag $f tokutek-mysql-build $f
exitcode=$?
echo `date` s3copykey $s3_release_bucket-$git_tag $f $exitcode
done
fi
fi
}
pushd $(dirname $0)
source ./common.sh
popd
PATH=$HOME/bin:$PATH
......@@ -665,14 +36,9 @@ system=$(uname -s | tr '[:upper:]' '[:lower:]')
arch=$(uname -m | tr '[:upper:]' '[:lower:]')
makejobs=$(get_ncpus)
github_use_ssh=0
git_tag=HEAD
mysqlbuild=
mysql=mysql-5.5.30
do_s3=0
s3_build_bucket=tokutek-mysql-build
s3_release_bucket=tokutek-mysql
do_make_check=0
mysql=
cc=gcc47
cxx=g++47
build_debug=0
......@@ -689,6 +55,7 @@ jemalloc_version=3.3.0
jemalloc_tree=
backup_tree=
# parse arguments
while [ $# -gt 0 ] ; do
arg=$1; shift
if [[ $arg =~ --(.*)=(.*) ]] ; then
......@@ -706,6 +73,7 @@ while [ $# -gt 0 ] ; do
fi
done
# compute more version names etc.
if [ -z $mysqlbuild ] ; then
if [ -z $tokudb_version ] ; then
if [ $git_tag = HEAD ] ; then
......@@ -726,33 +94,111 @@ if [ -z $mysqlbuild ] ; then
fi
fi
builddir=build-tokudb-$tokudb_version
if [ -d builds ] ; then builddir=builds/$builddir; fi
if [ ! -d $builddir ] ; then mkdir $builddir; fi
pushd $builddir
# download all the mysql source
if [ ! -d $mysql_distro ] ; then
github_download Tokutek/$mysql_distro $(git_tree $git_tag $mysql_tree) $mysql_distro
fi
basedir=$PWD
cd $mysql_distro
# build the fractal tree tarball
tokufractaltree=tokufractaltreeindex-$tokudb_version
tokuportability=tokuportability-$tokudb_version
tokufractaltreedir=$tokufractaltree-$system-$arch
build_fractal_tree
# install the backup source
if [ ! -d toku_backup ] ; then
github_download Tokutek/backup-$build_type $(git_tree $git_tag $backup_tree) backup-$build_type
cp -r backup-$build_type/backup toku_backup
fi
# build the mysql source tarball
mysqldir=$mysql-tokudb-$tokudb_version-$system-$arch
mysqlsrc=$mysql-tokudb-$tokudb_version
build_mysql_src
if [ ! -d ft-engine ] ; then
github_download Tokutek/ft-engine $(git_tree $git_tag $ftengine_tree) ft-engine
# install the tokudb storage engine source
cp -r ft-engine/storage/tokudb storage/
# build the mysql release tarball
mysqlbuild=$mysqldir-build
build_mysql_release
# merge the mysql tests
mv mysql-test mysql-test-save
cp -r ft-engine/mysql-test .
cp -r mysql-test-save/* mysql-test
rm -rf mysql-test-save
# copy to s3
if [ $do_s3 != 0 ] ; then
copy_to_s3 $s3_build_bucket $s3_release_bucket
# install the tokudb scripts
cp -r ft-engine/scripts/* scripts/
fi
popd
if [ ! -d storage/tokudb/ft-index ] ; then
github_download Tokutek/ft-index $(git_tree $git_tag $ftindex_tree) storage/tokudb/ft-index
fi
if [ ! -d storage/tokudb/ft-index/third_party/jemalloc ] ; then
github_download Tokutek/jemalloc $(git_tree $git_tag $jemalloc_tree) storage/tokudb/ft-index/third_party/jemalloc
fi
# append tokudb-specific version
if [ $(fgrep tokudb VERSION | wc -l) = 0 ] ; then
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
sed --in-place="" -e "s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${tokudb_version}/" VERSION
fi
# prints a cmake command to eval
function generate_cmake_cmd () {
local ft_revision=0x$(git ls-remote https://github.com/Tokutek/ft-index.git $git_tag | cut -c-7)
echo -n CC=$cc CXX=$cxx cmake \
-D BUILD_CONFIG=mysql_release \
-D CMAKE_BUILD_TYPE=$cmake_build_type \
-D CMAKE_TOKUDB_REVISION=$ft_revision \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
-D USE_CTAGS=OFF \
-D USE_ETAGS=OFF \
-D USE_CSCOPE=OFF
exit 0
if [ $build_debug = 1 ] ; then
echo -n " " \
-D USE_VALGRIND=ON \
-D TOKU_DEBUG_PARANOID=ON
else
echo -n " " \
-D USE_VALGRIND=OFF \
-D TOKU_DEBUG_PARANOID=OFF
fi
if [[ $mysql_distro =~ ^Percona ]] ; then
echo -n " " \
-D WITH_EMBEDDED_SERVER=OFF
fi
if [ $system = darwin ] ; then
echo -n " " \
-D WITH_SAFEMALLOC=OFF \
-D WITH_SSL=system
fi
if [ $build_type = enterprise ] ; then
echo -n " " \
-D COMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
if [ $system = linux -a $build_rpm != 0 -a $mysql_distro = mariadb ] ; then
linux_distro=linux
if [ -f /etc/issue ] ; then
if [[ "$(head -1 /etc/issue)" =~ "Red Hat Enterprise Linux Server release ([56])" ]] ; then
linux_distro=rhel${BASH_REMATCH[1]}
fi
if [[ "$(head -1 /etc/issue)" =~ "CentOS release ([56])" ]] ; then
linux_distro=centos${BASH_REMATCH[1]}
fi
fi
echo -n " " -D RPM=$linux_distro
fi
}
mkdir -p build.$cmake_build_type
cd build.$cmake_build_type
# actually build
eval $(generate_cmake_cmd) ..
make package -j$makejobs
if [ $system = linux -a $build_rpm != 0 -a $mysql_distro != mariadb ] ; then
echo 1>&2 "I don't know how to build rpms for mysql yet."
exit 1
fi
scripts/make.mysql.new.bash deleted 100755 → 0
View file @ 805a0c7d
#!/bin/bash
set -e
set -u
function usage() {
echo "make mysql with the tokudb storage engine from github repo's"
echo "--git_tag=$git_tag"
echo "--mysqlbuild=$mysqlbuild"
echo "--mysql=$mysql"
echo "--tokudb_version=$tokudb_version"
echo
echo "community release builds using the tokudb-7.0.1 git tag"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.0.1-linux-x86_64"
echo " make.mysql.bash --mysqlbuild=mariadb-5.5.30-tokudb-7.0.1-linux-x86_64"
echo " make.mysql.bash --git_tag=tokudb-7.0.1 --mysql=mysql-5.5.30"
echo
echo "community debug builds using the tokudb-7.0.1 git tag"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.0.1-debug-linux-x86_64"
echo
echo "enterprise release builds at the HEAD of the repos"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-test-e-linux-x86_64"
echo
echo "community release builds of a branch"
echo " make.mysql.bash --mysql=mysql-5.5.30 --mysql_tree=<your mysql tree name> --ftengine_tree=<your ft-engine tree name> --tokudb_version=<your test string>>"
return 1
}
pushd $(dirname $0)
source ./common.sh
popd
PATH=$HOME/bin:$PATH
system=$(uname -s | tr '[:upper:]' '[:lower:]')
arch=$(uname -m | tr '[:upper:]' '[:lower:]')
makejobs=$(get_ncpus)
git_tag=HEAD
mysqlbuild=
mysql=
cc=gcc47
cxx=g++47
build_debug=0
build_type=community
build_tgz=1
build_rpm=0
tokudb_version=
tokudb_patches=1
cmake_build_type=RelWithDebInfo
mysql_tree=
ftengine_tree=
ftindex_tree=
jemalloc_version=3.3.0
jemalloc_tree=
backup_tree=
# parse arguments
while [ $# -gt 0 ] ; do
arg=$1; shift
if [[ $arg =~ --(.*)=(.*) ]] ; then
k=${BASH_REMATCH[1]}; v=${BASH_REMATCH[2]}
eval $k=$v
if [ $k = mysqlbuild ] ; then
parse_mysqlbuild $mysqlbuild
if [ $? != 0 ] ; then exit 1; fi
elif [ $k = mysql ] ; then
parse_mysql $mysql
if [ $? != 0 ] ; then exit 1; fi
fi
else
usage; exit 1;
fi
done
# compute more version names etc.
if [ -z $mysqlbuild ] ; then
if [ -z $tokudb_version ] ; then
if [ $git_tag = HEAD ] ; then
tokudb_version=$(date +%s)
elif [[ $git_tag =~ tokudb-(.*) ]] ; then
tokudb_version=${BASH_REMATCH[1]}
else
tokudb_version=$git_tag
git_tag=HEAD
fi
fi
if [ $build_debug != 0 ] ; then
if [ $cmake_build_type = RelWithDebInfo ] ; then cmake_build_type=Debug; fi
tokudb_version=$tokudb_version-debug
fi
if [ $build_type = enterprise ] ; then
tokudb_version=$tokudb_version-e
fi
fi
# download all the mysql source
if [ ! -d $mysql_distro ] ; then
github_download Tokutek/$mysql_distro $(git_tree $git_tag $mysql_tree) $mysql_distro
fi
cd $mysql_distro
# install the backup source
if [ ! -d toku_backup ] ; then
github_download Tokutek/backup-$build_type $(git_tree $git_tag $backup_tree) backup-$build_type
cp -r backup-$build_type/backup toku_backup
fi
if [ ! -d ft-engine ] ; then
github_download Tokutek/ft-engine $(git_tree $git_tag $ftengine_tree) ft-engine
# install the tokudb storage engine source
cp -r ft-engine/storage/tokudb storage/
# merge the mysql tests
mv mysql-test mysql-test-save
cp -r ft-engine/mysql-test .
cp -r mysql-test-save/* mysql-test
rm -rf mysql-test-save
# install the tokudb scripts
cp -r ft-engine/scripts/* scripts/
fi
if [ ! -d storage/tokudb/ft-index ] ; then
github_download Tokutek/ft-index $(git_tree $git_tag $ftindex_tree) storage/tokudb/ft-index
fi
if [ ! -d storage/tokudb/ft-index/third_party/jemalloc ] ; then
github_download Tokutek/jemalloc $(git_tree $git_tag $jemalloc_tree) storage/tokudb/ft-index/third_party/jemalloc
fi
# append tokudb-specific version
if [ $(fgrep tokudb VERSION | wc -l) = 0 ] ; then
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
sed --in-place="" -e "s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${tokudb_version}/" VERSION
fi
# prints a cmake command to eval
function generate_cmake_cmd () {
local ft_revision=0x$(git ls-remote https://github.com/Tokutek/ft-index.git $git_tag | cut -c-7)
echo -n CC=$cc CXX=$cxx cmake \
-D BUILD_CONFIG=mysql_release \
-D CMAKE_BUILD_TYPE=$cmake_build_type \
-D CMAKE_TOKUDB_REVISION=$ft_revision \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
-D USE_CTAGS=OFF \
-D USE_ETAGS=OFF \
-D USE_CSCOPE=OFF
if [ $build_debug = 1 ] ; then
echo -n " " \
-D USE_VALGRIND=ON \
-D TOKU_DEBUG_PARANOID=ON
else
echo -n " " \
-D USE_VALGRIND=OFF \
-D TOKU_DEBUG_PARANOID=OFF
fi
if [[ $mysql_distro =~ ^Percona ]] ; then
echo -n " " \
-D WITH_EMBEDDED_SERVER=OFF
fi
if [ $system = darwin ] ; then
echo -n " " \
-D WITH_SAFEMALLOC=OFF \
-D WITH_SSL=system
fi
if [ $build_type = enterprise ] ; then
echo -n " " \
-D COMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
if [ $system = linux -a $build_rpm != 0 -a $mysql_distro = mariadb ] ; then
linux_distro=linux
if [ -f /etc/issue ] ; then
if [[ "$(head -1 /etc/issue)" =~ "Red Hat Enterprise Linux Server release ([56])" ]] ; then
linux_distro=rhel${BASH_REMATCH[1]}
fi
if [[ "$(head -1 /etc/issue)" =~ "CentOS release ([56])" ]] ; then
linux_distro=centos${BASH_REMATCH[1]}
fi
fi
echo -n " " -D RPM=$linux_distro
fi
}
mkdir -p build.$cmake_build_type
cd build.$cmake_build_type
# actually build
eval $(generate_cmake_cmd) ..
make package -j$makejobs
if [ $system = linux -a $build_rpm != 0 -a $mysql_distro != mariadb ] ; then
echo 1>&2 "I don't know how to build rpms for mysql yet."
exit 1
fi
scripts/make.mysql.old.bash 0 → 100755
View file @ aa59bca6
#!/usr/bin/env bash
# build mysql with the tokudb storage engine and the fractal tree
#
# parse arguments
# checkout the fractal tree
# checkout jemalloc
# build the fractal tree
# checkout the mysql source
# checkout the tokudb storage engine
# checkout the tokudb backup source
# generate a mysql build script
# generate a mysql source tarball
# execute the build script
# copy build files to an amazon s3 bucket
shopt -s compat31 2> /dev/null
function usage() {
echo "make.mysql.bash - build mysql with the tokudb storage engine and the fractal tree"
echo "--git_tag=$git_tag"
echo "--mysqlbuild=$mysqlbuild"
echo "--tokudb_version=$tokudb_version"
echo "--mysql=$mysql --mysql_distro=$mysql_distro --mysql_version=$mysql_version"
echo "--build_type=$build_type"
echo "--build_debug=$build_debug"
echo "--build_tgz=$build_tgz"
echo "--build_rpm=$build_rpm"
echo "--cc=$cc --cxx=$cxx"
echo "--do_make_check=$do_make_check"
echo "--do_s3=$do_s3 --s3_build_bucket=$s3_build_bucket --s3_release_bucket=$s3_release_bucket"
echo
echo "community release builds using the tokudb-7.0.1 git tag"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.0.1-linux-x86_64"
echo " make.mysql.bash --mysqlbuild=mariadb-5.5.30-tokudb-7.0.1-linux-x86_64"
echo " make.mysql.bash --git_tag=tokudb-7.0.1 --mysql=mysql-5.5.30"
echo
echo "community debug builds using the tokudb-7.0.1 git tag"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-7.0.1-debug-linux-x86_64"
echo
echo "enterprise release builds at the HEAD of the repos"
echo " make.mysql.bash --mysqlbuild=mysql-5.5.30-tokudb-test-e-linux-x86_64"
echo
echo "community release builds of a branch"
echo " make.mysql.bash --mysql=mysql-5.5.30 --mysql_tree=<your mysql tree name> --ftengine_tree=<your ft-engine tree name> --tokudb_version=<your test string>>"
return 1
}
function retry() {
local cmd
local retries
local exitcode
cmd=$*
let retries=0
while [ $retries -le 10 ] ; do
echo `date` $cmd
bash -c "$cmd"
exitcode=$?
echo `date` $cmd $exitcode $retries
let retries=retries+1
if [ $exitcode -eq 0 ] ; then break; fi
sleep 10
done
test $exitcode = 0
}
function github_download() {
repo=$1; shift
rev=$1; shift
dest=$1; shift
mkdir $dest
if [ ! -z $github_token ] ; then
retry curl \
--header "Authorization:\\ token\\ $github_token" \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
if [ $? != 0 ] ; then return; fi
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
elif [ ! -z $github_user ] ; then
retry curl \
--user $github_user \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
if [ $? != 0 ] ; then return; fi
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
elif [ $github_use_ssh != 0 ] ; then
tempdir=$(mktemp -d -p $PWD)
retry git clone git@github.com:${repo}.git $tempdir
if [ $? != 0 ] ; then return; fi
pushd $tempdir
if [ $? != 0 ] ; then return; fi
git checkout $rev
if [ $? != 0 ] ; then return; fi
popd
# export the right branch or tag
(cd $tempdir ;
git archive \
--format=tar \
$rev) | \
tar --extract \
--directory $dest
if [ $? != 0 ] ; then return; fi
rm -rf $tempdir
else
retry curl \
--location https://api.github.com/repos/$repo/tarball/$rev \
--output $dest.tar.gz
tar --extract \
--gzip \
--directory=$dest \
--strip-components=1 \
--file $dest.tar.gz
if [ $? != 0 ] ; then return; fi
rm -f $dest.tar.gz
fi
}
# returns b if b is defined else returns a
function git_tree() {
local a=$1; shift
local b=$1; shift
if [ ! -z $b ] ; then
echo $b
else
echo $a;
fi
}
# compute the number of cpus in this system. used to parallelize the build.
function get_ncpus() {
if [ -f /proc/cpuinfo ]; then
grep bogomips /proc/cpuinfo | wc -l
elif [ $system = darwin ] ; then
sysctl -n hw.ncpu
else
echo 1
fi
}
# check out the fractal tree source from subversion, build it, and make the fractal tree tarballs
function build_fractal_tree() {
if [ ! -d ft-index ] ; then
# get the ft-index repo
github_download Tokutek/ft-index $(git_tree $git_tag $ftindex_tree) ft-index
if [ $? != 0 ] ; then exit 1; fi
# get jemalloc
github_download Tokutek/jemalloc $(git_tree $git_tag $jemalloc_tree) ft-index/third_party/jemalloc
if [ $? != 0 ] ; then exit 1; fi
# get the commit id of the ft-index repo
local ft_revision=0
if [ ! -z $github_token ] ; then
local ft_index_ls
ft_index_ls=$(git ls-remote https://$github_token:x-oauth-basic@github.com/Tokutek/ft-index.git $git_tag)
if [ $? != 0 ] ; then exit 1; fi
local ft_revision_hex
ft_revision_hex=0x$(echo $ft_index_ls | cut -c-7)
let ft_revision=$ft_revision_hex
fi
pushd ft-index
if [ $? != 0 ] ; then exit 1; fi
local ft_build_type=""
local use_valgrind=""
local debug_paranoid=""
if [[ $build_debug = 1 ]]; then
ft_build_type=Debug
use_valgrind="ON"
debug_paranoid="ON"
else
ft_build_type=Release
use_valgrind="OFF"
debug_paranoid="OFF"
fi
mkdir -p build
pushd build
if [ $? != 0 ] ; then exit 1; fi
#-D JEMALLOC_SOURCE_DIR=../../jemalloc \
CC=$cc CXX=$cxx cmake \
-D LIBTOKUDB=$tokufractaltree \
-D LIBTOKUPORTABILITY=$tokuportability \
-D CMAKE_TOKUDB_REVISION=$ft_revision \
-D CMAKE_BUILD_TYPE=$ft_build_type \
-D CMAKE_INSTALL_PREFIX=../../$tokufractaltreedir \
-D BUILD_TESTING=OFF \
-D USE_GTAGS=OFF \
-D USE_CTAGS=OFF \
-D USE_ETAGS=OFF \
-D USE_CSCOPE=OFF \
-D USE_VALGRIND=$use_valgrind \
-D TOKU_DEBUG_PARANOID=$debug_paranoid \
..
make install -j$makejobs
if [ $? != 0 ] ; then exit 1; fi
popd # build
popd # ft-index
pushd $tokufractaltreedir/examples
if [ $? != 0 ] ; then exit 1; fi
# test the examples
sed -ie "s/LIBTOKUDB = tokudb/LIBTOKUDB = $tokufractaltree/" Makefile
sed -ie "s/LIBTOKUPORTABILITY = tokuportability/LIBTOKUPORTABILITY = $tokuportability/" Makefile
if [ $system = darwin ] ; then
DYLD_LIBRARY_PATH=$PWD/../lib:$DYLD_LIBRARY_PATH make check CC=$cc
exitcode=$?
else
if [ $do_make_check != 0 ] ; then
make check CC=$cc
exitcode=$?
else
exitcode=0
fi
fi
echo `date` make check examples $tokufractaltree $exitcode
if [ $exitcode != 0 ] ; then exit 1; fi
make clean
popd
# make tarballs
tar czf $tokufractaltreedir.tar.gz $tokufractaltreedir
md5sum $tokufractaltreedir.tar.gz >$tokufractaltreedir.tar.gz.md5
md5sum --check $tokufractaltreedir.tar.gz.md5
fi
}
function generate_cmake_cmd() {
local mysqlsrc=$1; shift
local extra_flags=$*
if [[ $mysqlsrc =~ ^Percona ]] ; then extra_flags=-DWITH_EMBEDDED_SERVER=OFF; fi
echo -n cmake .. $extra_flags -DBUILD_CONFIG=mysql_release -DCMAKE_BUILD_TYPE=$cmake_build_type
if [ $system = linux ] ; then
echo -n " " \
-DTOKU_JEMALLOC_LIBRARY=\"-Wl,--whole-archive \$TOKUFRACTALTREE/lib/libjemalloc.a -Wl,-no-whole-archive\"
elif [ $system = darwin ] ; then
echo -n " " \
-DWITH_SAFEMALLOC=OFF -DWITH_SSL=system \
-DTOKU_JEMALLOC_LIBRARY=\"-Wl,-force_load -Wl,\$TOKUFRACTALTREE/lib/libjemalloc.a\"
else
exit 1
fi
if [ $build_type = enterprise ] ; then
echo -n " " -DCOMPILATION_COMMENT=\"TokuDB Enterprise Server \(GPL\)\"
fi
echo
}
# generate a script that builds from the mysql src tarball and the fractal tree tarball
function generate_build_from_src() {
local mysqlsrc=$1; local tokufractaltreedir=$2
pushd $mysqlsrc >/dev/null 2>&1
if [ $? = 0 ] ; then
echo '#/usr/bin/env bash'
# defaults
echo 'makejobs=1'
echo 'cc=gcc47'
echo 'cxx=g++47'
# parse arguments
echo 'while [ $# -gt 0 ] ; do'
echo ' arg=$1; shift'
echo ' if [[ $arg =~ --(.*)=(.*) ]] ; then'
echo ' eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}'
echo ' else'
echo ' break'
echo ' fi'
echo 'done'
# check tarballs
echo md5sum --check $mysqlsrc.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo md5sum --check $tokufractaltreedir.tar.gz.md5
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo "if [ ! -d $mysqlsrc ] ; then"
echo " tar xzf $mysqlsrc.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
echo "if [ ! -d $tokufractaltreedir ] ;then"
echo " tar xzf $tokufractaltreedir.tar.gz"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo fi
# setup environment variables
echo export TOKUDB_VERSION=$tokudb_version
echo export TOKUFRACTALTREE='$PWD'/$tokufractaltreedir
echo export TOKUDB_PATCHES=$tokudb_patches
echo export TOKUFRACTALTREE_LIBNAME=${tokufractaltree}_static
echo export TOKUPORTABILITY_LIBNAME=${tokuportability}_static
echo 'export CC=$cc; export CXX=$cxx'
echo "pushd $mysqlsrc"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [ ! -f CMakeLists.txt ] ; then exit 1; fi
if [ $build_tgz != 0 ] ; then
echo "mkdir build.$cmake_build_type; pushd build.$cmake_build_type"
echo 'if [ $? != 0 ] ; then exit 1; fi'
cmd=$(generate_cmake_cmd $mysqlsrc)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# fixup tarball name mismatch (if any)
echo "if [ ! -f $mysqldir.tar.gz ] ; then"
echo ' oldtb=$(ls *.gz)'
echo ' if [[ $oldtb =~ (.*).tar.gz ]] ; then oldmysqldir=${BASH_REMATCH[1]}; fi'
echo ' tar xzf $oldtb'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo ' mv $oldmysqldir' $mysqldir
echo " tar czf $mysqldir.tar.gz $mysqldir"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
# add ft files to the mysql tarball
echo tar xzf $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
# add ftdump to mysql tarball
echo "cp ../../$tokufractaltreedir/bin/ftdump $mysqldir/bin/tokuftdump"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# cleanup
echo rm $mysqldir.tar.gz
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo tar czf $mysqldir.tar.gz $mysqldir
echo 'if [ $? != 0 ] ; then exit 1; fi'
# generate the md5 file
echo "if [ -f $mysqldir.tar.gz ] ; then"
echo " md5sum $mysqldir.tar.gz >$mysqldir.tar.gz.md5"
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'fi'
echo popd
fi
# make RPMs
if [ $system = linux -a $build_rpm != 0 ]; then
echo "mkdir build.${cmake_build_type}.rpms; pushd build.${cmake_build_type}.rpms"
echo 'if [ $? != 0 ] ; then exit 1; fi'
if [[ $mysqlsrc =~ ^mariadb ]] ; then
echo linux_distro=linux
echo 'if [ -f /etc/issue ] ; then'
echo ' if [[ "$(head -1 /etc/issue)" =~ Red\ Hat\ Enterprise\ Linux\ Server\ release\ ([56]) ]] ; then linux_distro=rhel${BASH_REMATCH[1]}; fi'
echo ' if [[ "$(head -1 /etc/issue)" =~ CentOS\ release\ ([56]) ]] ; then linux_distro=centos${BASH_REMATCH[1]}; fi'
echo fi
cmd=$(generate_cmake_cmd $mysqlsrc -DRPM=\$linux_distro)
echo $cmd
echo 'if [ $? != 0 ] ; then exit 1; fi'
# make binary distribution package
echo 'make package -j$makejobs'
echo 'if [ $? != 0 ] ; then exit 1; fi'
else
echo 'mkdir -p {tmp,BUILD,RPMS,SOURCES,SPECS,SRPMS}'
# split myslqsrc into components
echo "if [[ $mysqlsrc =~ (.*)-([0-9]+\.[0-9]+\.[0-9]+.*)-tokudb-(.*) ]] ; then"
echo ' mysql_distro=${BASH_REMATCH[1]}'
echo ' mysql_version=${BASH_REMATCH[2]}'
echo ' mysql_distro_version=${mysql_distro}-${mysql_version}'
echo ' tokudb_distro=tokudb'
echo ' tokudb_version=${BASH_REMATCH[3]}'
echo ' tokudb_distro_version=${tokudb_distro}-${tokudb_version}'
echo else
echo ' exit 1'
echo fi
# copy the source tarball
echo "cp ../../$mysqlsrc.tar.gz SOURCES/$mysqlsrc.tar.gz"
echo 'if [ $? != 0 ] ; then exit 1; fi'
# hack on the spec file
echo 'specfile=${mysql_distro}.${mysql_version}.spec'
echo 'cp ../support-files/$specfile SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'sed -i -e"s/^%define mysql_version.*/&-${tokudb_distro_version}/" SPECS/$specfile'
echo 'sed -i -e"s/^%define release.*/%define release $(echo ${tokudb_distro_version}|tr - .)/" SPECS/$specfile'
# echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DEXTRA_VERSION=-${tokudb_distro_version}/" SPECS/$specfile'
# include README-TOKUDB in the RPM
echo 'sed -i -e"s/README/& %{src_dir}\/README-TOKUDB/" SPECS/$specfile'
# add jemalloc to the linker flags
echo 'sed -i -e"s/^\(.*-DMYSQL_SERVER_SUFFIX=.*\)$/& -DTOKU_JEMALLOC_LIBRARY=\"-Wl,--whole-archive \$\{TOKUFRACTALTREE\}\/lib\/libjemalloc.a -Wl,-no-whole-archive\"/" SPECS/$specfile'
# add the hot backup lib to the server package file list
echo 'sed -i -e"s/%{_datadir}\/mysql\/$/&\n%attr(755, root, root) %{_libdir}\/libHotBackup*.so/" SPECS/$specfile'
# build generic linux RPMs
echo 'export MYSQL_BUILD_MAKE_JFLAG=-j$makejobs'
echo 'rpmbuild -v --define="_topdir $PWD" --define="_tmppath $PWD/tmp" --define="src_base $mysql_distro" -ba SPECS/$specfile'
echo 'if [ $? != 0 ] ; then exit 1; fi'
# move the rpms
echo 'pushd RPMS/$(uname -m)'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ../..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
echo 'pushd SRPMS'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'mv *.rpm ..'
echo 'if [ $? != 0 ] ; then exit 1; fi'
echo 'popd'
fi
# generate the md5 file
echo 'rpmfiles=$(ls *.rpm)'
echo 'for x in $rpmfiles; do'
echo ' md5sum $x >$x.md5'
echo ' if [ $? != 0 ] ; then exit 1; fi'
echo 'done'
echo popd
fi
echo popd
popd >/dev/null 2>&1
fi
}
# checkout the mysql source, generate a build script, and make the mysql source tarball
function build_mysql_src() {
if [ ! -d $mysqlsrc ] ; then
# get the mysql repo
if [ ! -d $mysql_distro ] ; then
github_download Tokutek/$mysql_distro $(git_tree $git_tag $mysql_tree) $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
fi
# get the hot backup source
if [ ! -d backup-$build_type ] ; then
github_download Tokutek/backup-$build_type $(git_tree $git_tag $backup_tree) backup-$build_type
if [ $? != 0 ] ; then exit 1; fi
fi
# get the ft-engine repo
if [ ! -d ft-engine ] ; then
github_download Tokutek/ft-engine $(git_tree $git_tag $ftengine_tree) ft-engine
if [ $? != 0 ] ; then exit 1; fi
fi
# append the tokudb version to the MYSQL_VERSION_EXTRA variable in the VERSION file
sed -i "" -e"s/^MYSQL_VERSION_EXTRA=.*/&-tokudb-${tokudb_version}/" $mysqlsrc/VERSION
# add the backup source
mkdir $mysqlsrc/toku_backup
pushd backup-$build_type/backup
if [ $? != 0 ] ; then exit 1; fi
copy_files=$(ls CMakeLists.txt *.h *.cc *.cmake export.map)
cp $copy_files ../../$mysqlsrc/toku_backup
if [ $? != 0 ] ; then exit 1; fi
popd
# add the tokudb storage engine source
mkdir -p $mysqlsrc/storage/tokudb
pushd ft-engine/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
copy_files=$(ls CMakeLists.txt *.h *.cc)
cp $copy_files ../../../$mysqlsrc/storage/tokudb
if [ $? != 0 ] ; then exit 1; fi
popd
# merge the common mysql tests into the source
mv $mysqlsrc/mysql-test $mysqlsrc/mysql-test-save
cp -r ft-engine/mysql-test $mysqlsrc
cp -r $mysqlsrc/mysql-test-save/* $mysqlsrc/mysql-test
rm -rf $mysqlsrc/mysql-test-save
# add the common scripts into the source
cp -r ft-engine/scripts $mysqlsrc
# generate the tokudb.build.bash script from the mysql src and the fractal tree tarballs
generate_build_from_src $mysqlsrc $tokufractaltreedir >$mysqlsrc/scripts/tokudb.build.bash
# add README-TOKUDB
if [ ! -f $mysqlsrc/README-TOKUDB ] ; then
cp ft-index/README-TOKUDB $mysqlsrc
fi
sed -i -e's/FILES README DESTINATION/FILES README README-TOKUDB DESTINATION/' $mysqlsrc/CMakeLists.txt
# make the mysql src tarball
tar czf $mysqlsrc.tar.gz $mysqlsrc
if [ $? != 0 ] ; then exit 1; fi
md5sum $mysqlsrc.tar.gz >$mysqlsrc.tar.gz.md5
if [ $? != 0 ] ; then exit 1; fi
fi
}
# make the mysql release tarball
# by executing the build script in the mysql source tarball and compiling with the fractal tree tarball
function build_mysql_release() {
if [ ! -d $mysqlbuild ] ; then
mkdir $mysqlbuild
pushd $mysqlbuild
# create links to the mysql source and fractal tree tarballs
ln $basedir/$mysqlsrc.tar.gz
ln $basedir/$mysqlsrc.tar.gz.md5
ln $basedir/$tokufractaltreedir.tar.gz
ln $basedir/$tokufractaltreedir.tar.gz.md5
# extract the build script
tar xzf $mysqlsrc.tar.gz $mysqlsrc/scripts/tokudb.build.bash
if [ $? != 0 ] ; then exit 1; fi
cp $mysqlsrc/scripts/tokudb.build.bash .
if [ $? != 0 ] ; then exit 1; fi
rm -rf $mysqlsrc
# execute the build script
bash -x tokudb.build.bash --makejobs=$makejobs --cc=$cc --cxx=$cxx
if [ $? != 0 ] ; then exit 1; fi
# move the build files
pushd $mysqlsrc/build.${cmake_build_type}
if [ $? = 0 ] ; then
cp $mysqldir.tar.gz* $basedir; if [ $? != 0 ] ; then exit 1; fi
popd
fi
if [ $build_rpm != 0 ] ; then
pushd $mysqlsrc/build.${cmake_build_type}.rpms
if [ $? = 0 ] ; then
rpmfiles=$(ls *.rpm *.rpm.md5)
for x in $rpmfiles; do cp $x $basedir; if [ $? != 0 ] ; then exit 1; fi; done
popd
fi
fi
popd
fi
}
# split mysql into mysql_distro and mysql_version
function parse_mysql() {
local mysql=$1
if [[ $mysql =~ ^(mysql|mariadb)-(.*)$ ]] ; then
mysql_distro=${BASH_REMATCH[1]}
mysql_version=${BASH_REMATCH[2]}
if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
exitcode=0
else
exitcode=1
fi
test $exitcode = 0
}
# parse a mysqlbuild string and extract the mysql_distro, mysql_version, tokudb_distro, tokudb_version, target_system, and target_arch
# compute build_type, build_debug, and git_tag
function parse_mysqlbuild() {
local mysqlbuild=$1
local exitcode=0
if [[ $mysqlbuild =~ ((mysql|mariadb|percona\-server)-(.*))-((tokudb)-(.*))-(linux|darwin)-(x86_64|i386) ]] ; then
# extract distros and versions from the components
mysql=${BASH_REMATCH[1]}
mysql_distro=${BASH_REMATCH[2]}
mysql_version=${BASH_REMATCH[3]}
tokudb=${BASH_REMATCH[4]}
tokudb_distro=${BASH_REMATCH[5]}
tokudb_version=${BASH_REMATCH[6]}
target_system=${BASH_REMATCH[7]}
target_arch=${BASH_REMATCH[8]}
# verify targets
if [ $target_system != $system ] ; then exitcode=1; fi
if [ $target_arch != $arch ] ; then exitcode=1; fi
local temp_tokudb_version=$tokudb_version
# decode enterprise
if [[ $temp_tokudb_version =~ (.*)-e$ ]] ; then
build_type=enterprise
temp_tokudb_version=${BASH_REMATCH[1]}
else
build_type=community
fi
# decode debug
if [[ $temp_tokudb_version =~ (.*)-debug$ ]] ; then
build_debug=1
temp_tokudb_version=${BASH_REMATCH[1]}
cmake_build_type=Debug
else
build_debug=0
fi
# set tag or HEAD
if [[ $temp_tokudb_version =~ ^([0-9]+)\\.([0-9]+)\\.([0-9]+)$ ]] ; then
git_tag=tokudb-$temp_tokudb_version
else
git_tag=HEAD
# setup _tree defaults
if [ -z $mysql_tree ] ; then mysql_tree=$mysql_version; fi
if [ -z $jemalloc_tree ] ; then jemalloc_tree=$jemalloc_version; fi
fi
# 5.6 is temp in another repo
if [[ $mysql_distro = mysql && $mysql_version =~ ^5.6 ]] ; then mysql_distro=mysql56; fi
else
exitcode=1
fi
test $exitcode = 0
}
# copy build files to amazon s3
function copy_to_s3() {
local $s3_build_bucket=$1; local $s3_release_bucket=$2
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3put $s3_build_bucket $f
s3put $s3_build_bucket $f $f
exitcode=$?
# index the file by date
echo `date` s3put $s3_build_bucket $f $exitcode
d=$(date +%Y%m%d)
s3put $s3_build_bucket-date $d/$f /dev/null
exitcode=$?
echo `date` s3put $s3_build_bucket-date $d/$f $exitcode
done
if [[ $git_tag =~ tokudb-.* ]] ; then
s3mkbucket $s3_release_bucket-$git_tag
if [ $? = 0 ] ; then
files=$(ls $tokufractaltreedir.tar.gz* $mysqldir.tar.gz* $mysqlsrc.tar.gz* *.rpm*)
for f in $files; do
echo `date` s3copykey $s3_release_bucket-$git_tag $f
s3copykey $s3_release_bucket-$git_tag $f tokutek-mysql-build $f
exitcode=$?
echo `date` s3copykey $s3_release_bucket-$git_tag $f $exitcode
done
fi
fi
}
PATH=$HOME/bin:$PATH
system=$(uname -s | tr '[:upper:]' '[:lower:]')
arch=$(uname -m | tr '[:upper:]' '[:lower:]')
makejobs=$(get_ncpus)
github_use_ssh=0
git_tag=HEAD
mysqlbuild=
mysql=mysql-5.5.30
do_s3=0
s3_build_bucket=tokutek-mysql-build
s3_release_bucket=tokutek-mysql
do_make_check=0
cc=gcc47
cxx=g++47
build_debug=0
build_type=community
build_tgz=1
build_rpm=0
tokudb_version=
tokudb_patches=1
cmake_build_type=RelWithDebInfo
mysql_tree=
ftengine_tree=
ftindex_tree=
jemalloc_version=3.3.0
jemalloc_tree=
backup_tree=
while [ $# -gt 0 ] ; do
arg=$1; shift
if [[ $arg =~ --(.*)=(.*) ]] ; then
k=${BASH_REMATCH[1]}; v=${BASH_REMATCH[2]}
eval $k=$v
if [ $k = mysqlbuild ] ; then
parse_mysqlbuild $mysqlbuild
if [ $? != 0 ] ; then exit 1; fi
elif [ $k = mysql ] ; then
parse_mysql $mysql
if [ $? != 0 ] ; then exit 1; fi
fi
else
usage; exit 1;
fi
done
if [ -z $mysqlbuild ] ; then
if [ -z $tokudb_version ] ; then
if [ $git_tag = HEAD ] ; then
tokudb_version=$(date +%s)
elif [[ $git_tag =~ tokudb-(.*) ]] ; then
tokudb_version=${BASH_REMATCH[1]}
else
tokudb_version=$git_tag
git_tag=HEAD
fi
fi
if [ $build_debug != 0 ] ; then
if [ $cmake_build_type = RelWithDebInfo ] ; then cmake_build_type=Debug; fi
tokudb_version=$tokudb_version-debug
fi
if [ $build_type = enterprise ] ; then
tokudb_version=$tokudb_version-e
fi
fi
builddir=build-tokudb-$tokudb_version
if [ -d builds ] ; then builddir=builds/$builddir; fi
if [ ! -d $builddir ] ; then mkdir $builddir; fi
pushd $builddir
basedir=$PWD
# build the fractal tree tarball
tokufractaltree=tokufractaltreeindex-$tokudb_version
tokuportability=tokuportability-$tokudb_version
tokufractaltreedir=$tokufractaltree-$system-$arch
build_fractal_tree
# build the mysql source tarball
mysqldir=$mysql-tokudb-$tokudb_version-$system-$arch
mysqlsrc=$mysql-tokudb-$tokudb_version
build_mysql_src
# build the mysql release tarball
mysqlbuild=$mysqldir-build
build_mysql_release
# copy to s3
if [ $do_s3 != 0 ] ; then
copy_to_s3 $s3_build_bucket $s3_release_bucket
fi
popd
exit 0
scripts/tokutek.make.mysql.bash
View file @ aa59bca6
......@@ -80,7 +80,7 @@ if [ $? != 0 ] ; then exit 1; fi
pushd $build_dir
# make mysql
bash -x $HOME/github/ft-engine/scripts/make.mysql.new.bash $make_args
bash -x $HOME/github/ft-engine/scripts/make.mysql.bash $make_args
if [ $? != 0 ] ; then exit 1; fi
# generate md5 sums
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7