Commit e2a41f61 authored by Julien Muchembled's avatar Julien Muchembled

version up: slapos.recipe.build 0.44

parent 5dfd47d0
......@@ -72,48 +72,49 @@ gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64
java_home = ${zulu:location}
bazelrc = ${:_profile_base_location_}/bazelrc
script =
extract_dir = self.extract(self.download(self.options['url'], self.options['md5sum']))
install =
import os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
crosstool_path = os.path.join(extract_dir, 'tools', 'cpp', 'CROSSTOOL')
os.chmod(crosstool_path, 0644)
shutil.copy(self.options['bazel-crosstool-modified-file-path'],
shutil.copy(options['bazel-crosstool-modified-file-path'],
crosstool_path)
src_main_cpp_build_path = os.path.join(extract_dir, 'src', 'main', 'cpp', 'BUILD')
os.chmod(src_main_cpp_build_path, 0644)
shutil.copy(self.options['bazel-src-main-cpp-build-path'],
shutil.copy(options['bazel-src-main-cpp-build-path'],
src_main_cpp_build_path)
src_main_tools_build_path = os.path.join(extract_dir, 'src', 'main', 'tools', 'BUILD')
os.chmod(src_main_tools_build_path, 0644)
shutil.copy(self.options['bazel-src-main-tools-build-path'],
shutil.copy(options['bazel-src-main-tools-build-path'],
src_main_tools_build_path)
src_tools_singlejar_build_path = os.path.join(extract_dir, 'src', 'tools', 'singlejar', 'BUILD')
os.chmod(src_tools_singlejar_build_path, 0644)
shutil.copy(self.options['bazel-src-tools-singlejar-build-path'],
shutil.copy(options['bazel-src-tools-singlejar-build-path'],
src_tools_singlejar_build_path)
target_path = extract_dir+'/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java'
os.chmod(target_path, 0644)
call([self.buildout['bazel']['patch-binary'], '-p1', '-d', extract_dir, '-i', self.buildout['bazel']['patch-file-path']])
path = ':'.join((
self.options['unzip-bin'],
self.options['zip-bin'],
self.options['gcc-bin'],
options['unzip-bin'],
options['zip-bin'],
options['gcc-bin'],
os.environ['PATH']
))
env = {'JAVA_HOME':self.options['java_home'],
env = {'JAVA_HOME':options['java_home'],
'PATH':path,
'LD_LIBRARY_PATH':':'.join((
self.options['gcc-lib'],
self.options['gcc-lib64'],
options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LD_LIBRARY_PATH', '')
)),
'LDFLAGS':'-Wl,-rpath='+self.options['gcc-lib64'],
'CC':self.options['gcc-bin']+'/gcc',
'CXX':self.options['gcc-bin']+'/g++',
'BAZELRC':self.options['bazelrc'],
'LDFLAGS':'-Wl,-rpath='+options['gcc-lib64'],
'CC':options['gcc-bin']+'/gcc',
'CXX':options['gcc-bin']+'/g++',
'BAZELRC':options['bazelrc'],
'VERBOSE':'yes',
}
bin_dir = os.path.join(self.options['location'], 'bin')
bin_dir = os.path.join(location, 'bin')
os.makedirs(bin_dir)
call(['bash', 'compile.sh', 'compile'], cwd=extract_dir, env=env)
shutil.copy(os.path.join(extract_dir, 'output', 'bazel'),
......
......@@ -19,11 +19,7 @@ url = http://www.boa.org/boa-0.94.14rc21.tar.gz
md5sum = e24b570bd767a124fcfb40a34d148ba9
patches =
${boa-patch-ENOSYS:location}/${boa-patch-ENOSYS:filename}
slapos_promise =
directory:bin/
file:bin/boa
file:bin/boa_indexer
script =
install =
import shutil
import os
url = self.download(self.options['url'], self.options['md5sum'])
......@@ -33,10 +29,11 @@ script =
call(['./configure'], cwd=workdir)
call(['make'], cwd=workdir)
# Installation of boa. Manually, no make install
os.makedirs('%(location)s/bin/')
shutil.copyfile(workdir + '/src/boa', '%(location)s/bin/boa')
os.chmod('%(location)s/bin/boa', 0755)
shutil.copyfile(workdir + '/src/boa_indexer', '%(location)s/bin/boa_indexer')
os.chmod('%(location)s/bin/boa_indexer', 0755)
bindir = os.path.join(location, 'bin')
os.makedirs(bindir)
for name in 'boa', 'boa_indexer':
path = os.path.join(bindir, name)
shutil.copyfile(os.path.join(workdir, 'src', name), path)
os.chmod(path, 0755)
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -10,16 +10,17 @@ url = http://git.busybox.net/busybox/snapshot/busybox-1_20_2.tar.gz
md5sum = 025acebb48040ef62dd635d416d317e8
patches =
${:_profile_base_location_}/busybox-1_20_2.patch#292498db86c46e101bb14bf2c74c36f0
script =
extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
install =
env = self.environ
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options.get('patches'), '-p1', cwd=workdir)
self.applyPatchList(options.get('patches'), '-p1', cwd=workdir)
self.logger.info("Creating default configuration")
call(['make', 'defconfig'], cwd=workdir, env=env)
self.logger.info("Building")
call(['make'], cwd=workdir, env=env)
self.logger.info("Installing")
call(['make', 'CONFIG_PREFIX=%(location)s', 'install'], cwd=workdir, env=env)
call(['make', 'CONFIG_PREFIX=' + location, 'install'], cwd=workdir, env=env)
self.logger.info("Installation finished")
environment =
PATH=${patch:location}/bin:%(PATH)s
......@@ -26,10 +26,11 @@ part = ${chromedriver:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
chromedriver = self.options['part']
install =
import os
chromedriver = options['part']
part = self.buildout[os.path.basename(chromedriver)]
with open(%(location)r, 'w') as f:
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
......@@ -65,10 +66,9 @@ library =
${nss:location}/lib
${nspr:location}/lib
path =
script =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
os.mkdir(%(location)r)
shutil.copy(extract_dir + '/chromedriver', %(location)r)
install =
import os, shutil
extract_dir = self.extract(self.download(*options[guessPlatform()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
......@@ -46,10 +46,11 @@ part = ${chromium:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
chromeium = self.options['part']
part = self.buildout[os.path.basename(chromeium)]
with open(%(location)r, 'w') as f:
install =
import os
chromium = options['part']
part = self.buildout[os.path.basename(chromium)]
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
# this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
......@@ -59,7 +60,7 @@ script =
export PATH={}:$PATH
exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
""".format(
chromeium,
chromium,
':'.join(part['library'].split()),
':'.join(part['path'].split()),
))
......@@ -103,9 +104,6 @@ md5sum-x86_64 =
recipe = slapos.recipe.build
slapos_promise =
file:chrome
x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
......@@ -153,13 +151,12 @@ library =
${zlib:location}/lib
path =
${fontconfig:location}/bin
script =
install =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), %(location)r)
self.copyTree(guessworkdir(extract_dir), location)
# XXX adjust some permissions
import os
os.system('"${findutils:location}/bin/find" "%(location)s" -type d -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
os.system('"${findutils:location}/bin/find" "%(location)s" -type f -executable -exec "${coreutils:location}/bin/chmod" a+rx {} \;')
os.system('"${findutils:location}/bin/find" "%(location)s" -type f -exec "${coreutils:location}/bin/chmod" a+r {} \;')
os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
......@@ -38,9 +38,8 @@ depends =
x86 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/28b36a94ad1a405bac689a76b6c353a7/condor-7.9.5-x86_Debian6-unstripped.tar.gz 227059bb9bebc9033665bb246d90a7bb
x86-64 = http://parrot.cs.wisc.edu//symlink/20130530031504/7/7.9/7.9.5/7000a12f9a22765b457e2958762871d5/condor-7.9.5-x86_64_Debian6-stripped.tar.gz 30fa4cebabb8ff4971c769f4ef74ba68
script =
if not self.options.get('path'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = self.options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
\ No newline at end of file
self.copyTree(workdir, location)
......@@ -15,15 +15,10 @@ url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
install =
import shutil
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
......@@ -31,10 +31,11 @@ part = ${firefox:location}
recipe = slapos.recipe.build
location = ${buildout:bin-directory}/${:wrapper-name}
script =
firefox = self.options['part']
install =
import os
firefox = options['part']
part = self.buildout[os.path.basename(firefox)]
with open(%(location)r, 'w') as f:
with open(location, 'w') as f:
f.write("""#!/bin/sh -e
cd {}
export LD_LIBRARY_PATH=$PWD:{}
......@@ -157,10 +158,10 @@ library =
path =
${fontconfig:location}/bin
script =
url, md5sum = self.options[guessPlatform()].split()
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
self.copyTree(guessworkdir(extract_dir), %(location)r)
self.copyTree(guessworkdir(extract_dir), location)
[geckodriver]
......@@ -218,7 +219,8 @@ location = ${buildout:bin-directory}/${:_buildout_section_name_}
x86 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
x86-64 = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
script =
url, md5sum = self.options[guessPlatform()].split()
install =
import shutil
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
shutil.copy(extract_dir + '/geckodriver', %(location)r)
shutil.copy(extract_dir + '/geckodriver', location)
......@@ -63,19 +63,25 @@ md5sum = d0efec10b9f110a32e9b8f796e21782c
[msttcore-fonts]
location = ${fonts:location}/${:_buildout_section_name_}
recipe = slapos.recipe.build
script =
install =
import os, subprocess
from zc.buildout.download import Download
d = self.options['location']
d = location
fonts = []
download = lambda x, dl=Download(self.buildout['buildout']): (
dl("http://downloads.sf.net/corefonts/%%s32.exe" %% name, md5sum=md5sum)
dl("http://downloads.sf.net/corefonts/%s32.exe" % name, md5sum=md5sum)
for md5sum, name in (x.split() for x in x.splitlines() if x))
extract = lambda x, d=d, p7z="${p7zip:location}/bin/7z": any(
subprocess.check_call((p7z, "x", "-ssc-", path, "*.ttf"), cwd=d)
for path, is_temp in x)
try: fonts += download(self.options['fonts']); os.makedirs(d); extract(fonts)
except: shutil.rmtree(d, ignore_errors=True); raise
finally: any(os.remove(path) for path, is_temp in fonts if is_temp)
try:
fonts += download(options['fonts'])
os.makedirs(d)
extract(fonts)
finally:
for path, is_temp in fonts:
if is_temp:
os.remove(path)
slapos_promise =
slapos_update_promise = ${:slapos_promise}
fonts =
......
......@@ -21,11 +21,11 @@ slapos_promisee =
# http://java.com/en/download/manual_java7.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
self.copyTree(workdir, location)
[java-re-8]
recipe = slapos.recipe.build
......@@ -40,11 +40,11 @@ slapos_promisee =
# Update 161
x86 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230530_2f38c3b165be4555a1fa6e98c45e0808 32db95dd417fd7949922206b2a61aa19
x86-64 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230532_2f38c3b165be4555a1fa6e98c45e0808 4385bc121b085862be623f4a31e7e0b4
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
install =
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
self.copyTree(workdir, location)
[java-re-8-output]
# Shared binary location to ease migration
......
......@@ -8,13 +8,12 @@ parts = jsl
recipe = slapos.recipe.build
url = http://www.javascriptlint.com/download/jsl-0.3.0-src.tar.gz
md5sum = 2b94ffa4fab07acabe0c5e73cd49bcdf
script =
location = %(location)r
self.failIfPathExists(location)
install =
import os
import sys
url = self.options['url']
md5sum = self.options['md5sum']
url = options['url']
md5sum = options['md5sum']
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
os.chdir(os.path.join(workdir, 'src'))
os.system('make -f Makefile.ref -j 1 DIST="%(location)s" all export')
os.system("make -f Makefile.ref -j 1 DIST='%s' all export" % location)
......@@ -7,7 +7,6 @@ parts = libcap
[libcap2]
recipe = slapos.recipe.build
format = no
url = http://pkgs.fedoraproject.org/lookaside/pkgs/libcap/libcap-2.22.tar.bz2/ce64058bdb3f086ddbfca8ce6c919845/libcap-2.22.tar.bz2
md5sum = ce64058bdb3f086ddbfca8ce6c919845
attr-include = ${attr:location}/include/
......@@ -19,17 +18,17 @@ slapos_promise =
file:lib/libcap.so
file:sbin/getcap
file:sbin/setcap
script =
install =
import os
url = self.download(self.options['url'], self.options['md5sum'])
url = self.download(options['url'], options['md5sum'])
extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir)
cflags = '-I%(attr)s' % {'attr': self.options['attr-include']}
ldflags = '-L%(attr)s -Wl,-rpath=%(attr)s' % {'attr': self.options['attr-lib']}
call(['make', 'CFLAGS=%s' % cflags, 'LDFLAGS=%s' % ldflags, 'DESTDIR=%s' % self.options['location'], 'RAISE_SETFCAP=no', 'prefix=', 'install'],
cwd=workdir, env=env)
lib64 = os.path.join(self.options['location'], 'lib64')
lib = os.path.join(self.options['location'], 'lib')
cflags = '-I%(attr-include)s' % options
ldflags = '-L%(attr-lib)s -Wl,-rpath=%(attr-lib)s' % options
call(['make', 'CFLAGS=' + cflags, 'LDFLAGS=' + ldflags, 'DESTDIR=' + location, 'RAISE_SETFCAP=no', 'prefix=', 'install'],
cwd=workdir, env=self.environ)
lib64 = os.path.join(location, 'lib64')
lib = os.path.join(location, 'lib')
# XXX: Dirty if case
# if lib64 exists, then create a symlink from lib to lib64
os.path.exists(lib64) and os.symlink(lib64, lib)
......
......@@ -17,28 +17,31 @@ md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
# where office code can be found?
officedir = libreoffice5.2
# script to install
script =
location = %(location)r
self.failIfPathExists(location)
install =
import os
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
platform = guessPlatform()
url = self.options['url'] %% (ARCH_DIR_MAP[platform], platform)
md5sum = self.options['md5sum_' + platform]
url = options['url'] % (ARCH_DIR_MAP[platform], platform)
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
storagedir = os.path.join(workdir, 'storage')
os.mkdir(storagedir)
rpmsdir = os.path.join(workdir, [q for q in os.listdir(workdir) if q == 'RPMS'][0])
rpmlist = [os.path.join(rpmsdir, q) for q in os.listdir(rpmsdir) if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q]
[self.pipeCommand([[sys.executable, '${:rpm2cpio}', rpm], ['${:cpio}', '-idum']], cwd=storagedir) for rpm in rpmlist]
self.copyTree(os.path.join(storagedir, 'opt', '${:officedir}'), location, ['ure-link'])
rpmsdir = os.path.join(workdir, 'RPMS')
for q in os.listdir(rpmsdir):
if q.endswith('.rpm') and 'javafilter' not in q and 'xsltfilter' not in q:
self.pipeCommand((
(sys.executable, options['rpm2cpio'], os.path.join(rpmsdir, q)),
(options['cpio'], '-idum'),
), cwd=storagedir)
self.copyTree(os.path.join(storagedir, 'opt', options['officedir']),
location, ['ure-link'])
os.symlink('ure', os.path.join(location, 'ure-link'))
# backward compatibility for cloudooo configuration
os.mkdir(os.path.join(location, 'basis-link'))
os.symlink(os.path.join('..', 'program'), os.path.join(location, 'basis-link', 'program'))
os.symlink(os.path.join('..', 'program'),
os.path.join(location, 'basis-link', 'program'))
# helper binaries
cpio = ${cpio:location}/bin/cpio
......
......@@ -15,15 +15,10 @@ version = 0.7.5
md5sum_x86 = a545108a0ccfde7c1e74de6c4e6fdded
md5sum_x86-64 = f343d709b84db494e8d6ec38259aa4a6
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': '386', 'x86-64': 'amd64' }
install =
import shutil
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
......@@ -17,20 +17,17 @@ depends =
x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686.tar.bz2 9c1426eef5b04679d65198b1bdd6ef88
x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
script =
if not self.options.get('url'):
self.options['url'], self.options['md5sum'] = \
self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'],
self.options.get('md5sum')))
install =
import os
url, md5sum = options[guessPlatform()].split()
extract_dir = self.extract(self.download(url, md5sum))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
with open(wrapper_location, 'w') as wrapper:
wrapper.write("""#!/bin/sh
self.copyTree(workdir, location)
with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
wrapper.write("""#!/bin/sh -e
cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/phantomjs "$@"
""")
os.chmod(wrapper_location, 0o755)
""" % options)
os.fchmod(wrapper.fileno(), 0o755)
......@@ -69,8 +69,8 @@ filename = get-pip.py
mode = 0755
[pyodide-script]
# XXX: A slapos.recipe.build section must not any FS change outside location.
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
build_dir = ${buildout:parts-directory}/pyodide-script-test
part_dir = ${buildout:parts-directory}
git-executable = ${git:location}/bin/git
......@@ -87,28 +87,26 @@ gcc_bin_dir = ${gcc-8.2:location}/bin
python_bin_dir = ${python3.7:location}/bin
md5sum = 47ec6a091e503349d948760dc03bed51
pip_script = ${get-pip:target}
script =
install =
import subprocess, os, shutil
env = {
'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
'CPATH':self.options['CPATH'],
'CPPFLAGS':self.options['CPPFLAGS'],
'LDFLAGS':self.options['LDFLAGS'],
'PKG_CONFIG_PATH':options['PKG_CONFIG_PATH'],
'PATH':options['PATH'] + ':' + os.environ['PATH'],
'CPATH':options['CPATH'],
'CPPFLAGS':options['CPPFLAGS'],
'LDFLAGS':options['LDFLAGS'],
}
if not os.path.isdir(self.options['build_dir']):
command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
# put the Popen in the loop to let the pipe.wait() work.
if not os.path.isdir(options['build_dir']):
command_list_0 = [{'cmd': options['git-executable'] + ' clone ' + options['repository'] + ' pyodide-script-test', 'cwd': options['part_dir']}]
for command_0 in command_list_0:
pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
# else:
# shutil.rmtree(self.options['build_dir'], ignore_errors=True)
# shutil.rmtree(options['build_dir'], ignore_errors=True)
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
pip_script = self.options['pip_script']
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + options['openssl_location'] + ' --prefix@" ' + options['build_dir'] + '/cpython/Makefile'
pip_script = options['pip_script']
install_pip = "python3 " + pip_script
install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
# Hack: create a gfortran symlink inside the python bin dir.
......@@ -119,28 +117,27 @@ script =
# So we are not going to provide the custom gcc bin path in the PATH environment variable.
# But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
# Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
create_gfortran_symlink = 'ln -s ' + options['gcc_bin_dir'] + '/gfortran ' + options['python_bin_dir'] + '/gfortran'
command_list = [
{'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
{'cmd': install_pip, 'cwd': self.options['build_dir']},
{'cmd': install_pytest, 'cwd': self.options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
{'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
{'cmd': cpython_makefile_command, 'cwd': options['build_dir']},
{'cmd': install_pip, 'cwd': options['build_dir']},
{'cmd': install_pytest, 'cwd': options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': options['build_dir']},
{'cmd': 'make -C packages', 'cwd': options['build_dir']},
]
for command in command_list:
pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command['cmd'], cwd=command['cwd'], env=env, shell=True)
# We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
# Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
emsdk_env_content = open(self.options['emsdk_set_env_script'])
emsdk_env_content = open(options['emsdk_set_env_script'])
for content in emsdk_env_content:
# remove "export " prefix
content = content[7:]
......@@ -153,13 +150,12 @@ script =
env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
command_list_2 = [
{'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
{'cmd': 'make -e', 'cwd': self.options['build_dir']},
{'cmd': 'make build/pyodide.asm.js', 'cwd': options['build_dir']},
{'cmd': 'make -e', 'cwd': options['build_dir']},
]
for command_2 in command_list_2:
pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
pipe.wait()
subprocess.check_call(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
[python3.7-PyYAML]
recipe = slapos.recipe.cmmi
......
......@@ -32,15 +32,15 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[python-cocoapi-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${python-cocoapi-repository:location}/PythonAPI
python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
gcc-location = ${gcc:location}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
gcc_location = self.options['gcc-location']
workdir = options['workdir']
python_bin = options['python-bin']
gcc_location = options['gcc-location']
env = {'PATH':':'.join([gcc_location+'/bin',
os.environ['PATH']]),
'CC':gcc_location+'/bin/gcc',
......@@ -50,16 +50,17 @@ script =
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[python-cocoapi-build-install-egg]
# XXX: A slapos.recipe.build section must not any FS change outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
python-cocoapi-repository-path = ${python-cocoapi-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-python-cocoapi-build = ${python-cocoapi-build:location}
egg = pycocotools
script =
install =
import os
os.makedirs(location)
workdir = self.options['python-cocoapi-repository-path']+'/PythonAPI'
workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
......@@ -44,7 +44,6 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pytorch-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${pytorch-repository:location}
python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
no-cuda = 0
......@@ -53,16 +52,17 @@ git-bin = ${git:location}/bin
binutils-location = ${binutils:location}
gcc-location = ${gcc:location}
openblas-location = ${openblas:location}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
python_bin = self.options['python-bin']
binutils_location = self.options['binutils-location']
gcc_location = self.options['gcc-location']
openblas_location = self.options['openblas-location']
workdir = options['workdir']
python_bin = options['python-bin']
binutils_location = options['binutils-location']
gcc_location = options['gcc-location']
openblas_location = options['openblas-location']
env = {'PYTHONPATH':workdir,
'PATH':':'.join([self.options['cmake-bin'],
self.options['git-bin'],
'PATH':':'.join([options['cmake-bin'],
options['git-bin'],
binutils_location+'/bin',
gcc_location+'/bin',
os.environ['PATH']]),
......@@ -84,26 +84,27 @@ script =
'-Wl,-rpath,'+openblas_location+'/lib',
])
}
if self.options.get('no-cuda') == '1':
if options.get('no-cuda') == '1':
env['NO_CUDA'] = '1'
import os.path
call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[pytorch-build-install-egg]
# XXX: A slapos.recipe.build section must not any FS change outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
pytorch-repository-path = ${pytorch-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-pytorch-build = ${pytorch-build:location}
egg = torch
script =
install =
import os
os.makedirs(location)
workdir = self.options['pytorch-repository-path']
workdir = options['pytorch-repository-path']
egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[pytorch-egg]
recipe = zc.recipe.egg
......
......@@ -33,13 +33,9 @@ library =
${libffi:location}/lib
${zlib:location}/lib
${libuuid:location}/lib
script =
import glob
location = %(location)r
self.failIfPathExists(location)
url = self.options['url']
md5sum = self.options['md5sum']
extract_dir = self.extract(self.download(url, md5sum))
install =
import glob, os, shutil
extract_dir = self.extract(self.download(options['url'], options['md5sum']))
os.mkdir(location)
source_file = glob.glob(os.path.join(extract_dir, '*', 'client-linux-debug'))[0]
shutil.copy(source_file, location)
......@@ -49,7 +45,7 @@ script =
export LD_LIBRARY_PATH={}
exec {}/client-linux-debug "$@"
""".format(
':'.join(self.options['library'].split()),
':'.join(options['library'].split()),
location
))
os.fchmod(f.fileno(), 0o755)
......@@ -6,17 +6,13 @@ parts = spidermonkey
recipe = slapos.recipe.build
url = http://ftp.mozilla.org/pub/mozilla.org/js/js-1.7.0.tar.gz
md5sum = 5571134c3863686b623ebe4e6b1f6fe6
slapos_promise =
directory:include/
directory:bin/
file:bin/js
statlib:lib/libjs.a
file:lib/libjs.so
script =
url = self.download(self.options['url'], self.options['md5sum'])
install =
import os
url = self.download(options['url'], options['md5sum'])
extract_dir = self.extract(url)
workdir = os.path.join(guessworkdir(extract_dir), 'src')
env.update(JS_DIST=self.options['location'])
env = self.environ
env['JS_DIST'] = location
call(['make', '-f', 'Makefile.ref'], cwd=workdir, env=env)
call(['make', '-f', 'Makefile.ref', 'export'], cwd=workdir, env=env)
......
......@@ -19,7 +19,6 @@ command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:
[tensorboard-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${tensorboard-repository:location}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
......@@ -28,22 +27,22 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel-bin = ${bazel:location}/bin
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
env = {'PATH':':'.join([self.options['gcc-bin'],
self.options['java_home_bin'],
self.options['bazel-bin'],
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
options['bazel-bin'],
os.environ['PATH']]),
'COMPILER_PATH':':'.join([self.options['gcc-bin'],
'COMPILER_PATH':':'.join([options['gcc-bin'],
os.environ.get('COMPILER_PATH') or '']),
'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
self.options['gcc-lib64'],
'LIBRARY_PATH':':'.join([options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LIBRARY_PATH') or '']),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_LIB_PATH':self.options['python27-lib'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
'PYTHON_LIB_PATH':options['python27-lib'],
}
import os.path
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
call(bazel_command, cwd=workdir, env=env)
......@@ -52,24 +51,24 @@ script =
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorboard-repository-path = ${tensorboard-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin
java_home_bin = ${bazel:java_home}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
install =
import os
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
workdir = options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
self.options['java_home_bin'],
env = {'PATH':':'.join([options['bazel-bin'],
options['java_home_bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
}
call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
[buildout]
# [buildout]
extends =
../../stack/slapos.cfg
../scipy/buildout.cfg
......@@ -43,7 +43,6 @@ cudnn_install_path = /usr/local/cuda
[tensorflow-build]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
workdir = ${tensorflow-repository:location}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
......@@ -53,20 +52,21 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel_bin = ${bazel:location}/bin
need_cuda = ${cuda:cuda_toolkit_path}
script =
install =
import os
os.makedirs(location)
workdir = self.options['workdir']
env = {'PATH':':'.join([self.options['gcc-bin'],
self.options['java_home_bin'],
self.options['bazel_bin'],
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
options['bazel_bin'],
os.environ['PATH']]),
'COMPILER_PATH':':'.join([self.options['gcc-bin'],
'COMPILER_PATH':':'.join([options['gcc-bin'],
os.environ.get('COMPILER_PATH') or '']),
'LIBRARY_PATH':':'.join([self.options['gcc-lib'],
self.options['gcc-lib64'],
'LIBRARY_PATH':':'.join([options['gcc-lib'],
options['gcc-lib64'],
os.environ.get('LIBRARY_PATH') or '']),
'PYTHON_BIN_PATH':self.options['numpy-python-command'],
'PYTHON_LIB_PATH':self.options['python27-lib'],
'PYTHON_BIN_PATH':options['numpy-python-command'],
'PYTHON_LIB_PATH':options['python27-lib'],
'CC_OPT_FLAGS':'-march=native',
'TF_NEED_JEMALLOC':'1',
'TF_ENABLE_XLA':'0',
......@@ -79,7 +79,7 @@ script =
'TF_NEED_MPI':'0',
'TF_NEED_CUDA':self.buildout['cuda']['tf_need_cuda'],
##### FOR CUDA #####
'GCC_HOST_COMPILER_PATH':os.path.join(self.options['gcc-bin'], 'gcc'),
'GCC_HOST_COMPILER_PATH':os.path.join(options['gcc-bin'], 'gcc'),
'TF_CUDA_VERSION':self.buildout['cuda']['tf_cuda_version'],
'CUDA_TOOLKIT_PATH':self.buildout['cuda']['cuda_toolkit_path'],
'TF_CUDNN_VERSION':self.buildout['cuda']['tf_cudnn_version'],
......@@ -87,10 +87,9 @@ script =
'TF_CUDA_COMPUTE_CAPABILITIES':self.buildout['cuda']['tf_cuda_compute_capabilities'],
####################
}
import os.path
if not os.path.exists(env['CUDA_TOOLKIT_PATH']): env['TF_NEED_CUDA'] = '0'
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
env['LDFLAGS'] = '-lW,-rpath='+self.options['gcc-lib64']
env['LDFLAGS'] = '-lW,-rpath='+options['gcc-lib64']
call(['./configure'], cwd=workdir, env=env)
bazel_command = (env['TF_NEED_CUDA'] == '1' and
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
......@@ -99,21 +98,20 @@ script =
call(bazel_command, cwd=workdir, env=env)
[tensorflow-build-install-egg]
# XXX: A slapos.recipe.build section must not any FS change outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorflow-repository-path = ${tensorflow-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-tensorboard-build = ${tensorboard-build:location}
need-tensorflow-build = ${tensorflow-build:location}
need-protobuf-python = ${protobuf-python:egg}
egg = tensorflow
script =
install =
import os
os.makedirs(location)
workdir = self.options['tensorflow-repository-path']
workdir = options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
call([self.options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
......@@ -23,33 +23,24 @@ url_x86 = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux
md5sum_x86 = ce1a2c0b2cf786ccc5d5828c42c99ddd
md5sum_x86-64 = 96b7306cebb9e65355f69f7ab63df68b
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
import os
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
install =
import os,shutil, sys, tempfile
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
url = options['url_' + platform]
md5sum = options['md5sum_' + platform]
path = self.download(url, md5sum)
import tempfile
extract_dir = tempfile.mkdtemp(self.name)
self.cleanup_dir_list.append(extract_dir)
self.logger.debug('Created working directory ' + repr(extract_dir))
self.logger.debug('Created working directory %s', extract_dir)
env = os.environ.copy()
env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
call(["tar", "xJf", path, "-C", extract_dir], env=env)
shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
wrapper_location = os.path.join("%(location)s", "wkhtmltopdf")
wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash
with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
wrapper.write("""#!/bin/sh
export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/bin/wkhtmltopdf "$@"
""")
wrapper.close()
os.chmod(wrapper_location, 0755)
""" % options)
os.fchmod(wrapper.fileno(), 0o755)
......@@ -8,9 +8,9 @@ list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
install =
with open(options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
i = src.index('[buildout]')
with open(location, 'w') as dst:
dst.write(src[:i] + options['extra'] + '\n' + src[i:])
......@@ -37,11 +37,10 @@ md5sum = 374e00dfbd28f27819ccfff31d169c9c
[gpg-decrypt]
recipe = slapos.recipe.build
gpg = ${gnupg:location}/bin/gpg
script =
import subprocess, zc.buildout
options = self.options
install =
import subprocess
args = (options['gpg'], '-d', '--batch', '--passphrase-fd', '0',
'-o', options['location'], options['input'])
'-o', location, options['input'])
p = subprocess.Popen(args, stdin=subprocess.PIPE)
p.communicate(options['password'])
retcode = p.poll()
......
......@@ -143,7 +143,7 @@ slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.4
slapos.recipe.build = 0.43
slapos.recipe.build = 0.44
slapos.recipe.cmmi = 0.12
slapos.toolbox = 0.109
stevedore = 1.21.0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment