Commit 0887b490 authored by Julien Muchembled's avatar Julien Muchembled

keras: some clean-up

For repositories with patches, the latter should be downloaded first.

Another issue that remains unfixed is that during the build of some
components, some data are cached in ~/.cache and this can break further
builds with the same user.
parent 92fdea86
......@@ -14,7 +14,7 @@ initialization =
eggs =
${scipy:egg}
${numpy:egg}
${tensorflow-build-install-egg:egg}
${tensorflow:egg}
${protobuf-python:egg}
${python-PyYAML:egg}
six
......@@ -25,6 +25,10 @@ interpreter = keras-python
scripts = keras-python
[versions]
backports.weakref = 1.0.post1
futures = 3.2.0
Keras = 2.1.0
tensorflow = 1.4.0
markdown = 3.1.1
protobuf = 3.11.3
h5py = 2.7.0rc2
wheel = 0.33.6
[buildout]
extends =
../../stack/slapos.cfg
../gcc/buildout.cfg
../cython/buildout.cfg
../scipy/buildout.cfg
parts =
python-cocoapi-build-install-egg
pycocotools
[python-cocoapi-repository]
recipe = plone.recipe.command
stop-on-error = true
recipe = slapos.recipe.build:gitclone
repository = https://github.com/cocodataset/cocoapi.git
tag = master
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/setup.py.patch ) || (rm -fr ${:location}; exit 1)
git-executable = ${git:location}/bin/git
revision = 0d86027d941482ae9649fb9e6e871704efc48b06
[python-cocoapi-build-interpreter]
recipe = zc.recipe.egg
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
eggs =
setuptools
[pycocotools]
recipe = zc.recipe.egg:develop
egg = ${:_buildout_section_name_}
setup = ${python-cocoapi-repository:location}/PythonAPI
environment = scipy-env
setup-eggs =
${cython:egg}
${scipy:egg}
${numpy:egg}
interpreter = python-cocoapi-build-interpreter
scripts = python-cocoapi-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[python-cocoapi-build]
recipe = slapos.recipe.build
workdir = ${python-cocoapi-repository:location}/PythonAPI
python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
gcc-location = ${gcc:location}
install =
import os
os.makedirs(location)
workdir = options['workdir']
python_bin = options['python-bin']
gcc_location = options['gcc-location']
env = {'PATH':':'.join([gcc_location+'/bin',
os.environ['PATH']]),
'CC':gcc_location+'/bin/gcc',
'CXX':gcc_location+'/bin/g++',
}
call([python_bin, 'setup.py', 'build_ext'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[python-cocoapi-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
python-cocoapi-repository-path = ${python-cocoapi-repository:location}
need-python-cocoapi-build = ${python-cocoapi-build:location}
egg = pycocotools
install =
import os
os.makedirs(location)
workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
${scipy:egg}
diff --git a/PythonAPI/setup.py b/PythonAPI/setup.py
index eb3d508..d619b3c 100644
--- a/PythonAPI/setup.py
+++ b/PythonAPI/setup.py
@@ -1,4 +1,5 @@
-from distutils.core import setup
+#from distutils.core import setup
+from setuptools import setup
from Cython.Build import cythonize
from distutils.extension import Extension
import numpy as np
@@ -21,4 +22,4 @@ setup(name='pycocotools',
version='2.0',
ext_modules=
cythonize(ext_modules)
- )
\ No newline at end of file
+ )
[buildout]
extends =
../../stack/slapos.cfg
../gcc/buildout.cfg
../openblas/buildout.cfg
../cmake/buildout.cfg
......@@ -10,20 +9,25 @@ extends =
../pillow/buildout.cfg
../scipy/buildout.cfg
../matplotlib/buildout.cfg
../unzip/buildout.cfg
parts =
pytorch-egg
torch
[pytorch-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/pytorch/pytorch
tag = master
commit = 4af66c43045a317b477918c503d105f565b4a66b
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recursive --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:git-binary} checkout ${:commit}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/pytorch.4af66c4.patch ) || (rm -fr ${:location}; exit 1)
command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git config submodule.recurse true
git reset --merge ${:commit}
git submodule update --init
git apply ${:_profile_base_location_}/pytorch.4af66c4.patch
) || { rm -fr ${:location}; exit 1; }
[pytorch-build-interpreter]
recipe = zc.recipe.egg
......@@ -42,9 +46,12 @@ interpreter = pytorch-build-interpreter
scripts = pytorch-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pytorch-build]
[torch]
recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${pytorch-repository:location}
egg-name = ${:egg}-${versions:torch}-py2.7-linux-x86_64.egg
location = ${buildout:eggs-directory}/${:egg-name}
python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
no-cuda = 0
cmake-bin = ${cmake:location}/bin
......@@ -53,8 +60,8 @@ binutils-location = ${binutils:location}
gcc-location = ${gcc:location}
openblas-location = ${openblas:location}
install =
import os
os.makedirs(location)
import os, shutil
from setuptools.archive_util import unpack_archive
workdir = options['workdir']
python_bin = options['python-bin']
binutils_location = options['binutils-location']
......@@ -86,25 +93,15 @@ install =
}
if options.get('no-cuda') == '1':
env['NO_CUDA'] = '1'
call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[pytorch-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
pytorch-repository-path = ${pytorch-repository:location}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
need-pytorch-build = ${pytorch-build:location}
egg = torch
install =
import os
os.makedirs(location)
workdir = options['pytorch-repository-path']
egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
os.chdir(workdir)
try:
call([python_bin, 'setup.py', 'build'], env=env)
call([python_bin, 'setup.py', 'bdist_egg'], env=env)
unpack_archive(os.path.join('dist', options['egg-name']), location)
finally:
for d in 'build', 'dist':
if os.path.exists(d):
shutil.rmtree(d)
[pytorch-egg]
recipe = zc.recipe.egg
......@@ -114,9 +111,9 @@ eggs =
${scipy:egg}
${numpy:egg}
${python-PyYAML:egg}
${pytorch-build-install-egg:egg}
${torch:egg}
${pillow-python:egg}
${python-cocoapi-build-install-egg:egg}
${pycocotools:egg}
${matplotlib:egg}
six
torchvision
......@@ -125,3 +122,4 @@ scripts = pytorch-python
[versions]
torchvision = 0.1.6
torch = 0.2.0+4af66c4
[buildout]
extends =
../zip/buildout.cfg
../bazel/buildout.cfg
../git/buildout.cfg
parts =
slapos-cookbook-develop
slapos-cookbook
tensorboard-build-install-egg
tensorflow_tensorboard
[tensorboard-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorboard
tag = 0.4.0
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
commit = ${versions:tensorflow-tensorboard}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git reset --merge ${:commit}
git apply ${:_profile_base_location_}/0.4.0.patch
) || { rm -fr ${:location}; exit 1; }
[tensorboard-build]
[tensorflow_tensorboard]
recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${tensorboard-repository:location}
egg-name = ${:egg}-${versions:tensorflow-tensorboard}-py2.7.egg
location = ${buildout:eggs-directory}/${:egg-name}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64
......@@ -28,8 +34,8 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel-bin = ${bazel:location}/bin
install =
import os
os.makedirs(location)
import os, shutil
from setuptools.archive_util import unpack_archive
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
......@@ -45,30 +51,16 @@ install =
}
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
call(bazel_command, cwd=workdir, env=env)
[tensorboard-build-install-egg]
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorboard-repository-path = ${tensorboard-repository:location}
need-tensorboard-build = ${tensorboard-build:location}
egg = tensorflow-tensorboard
bazel-bin = ${bazel:location}/bin
java_home_bin = ${bazel:java_home}/bin
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
install =
import os
os.makedirs(location)
workdir = options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([options['bazel-bin'],
options['java_home_bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':options['numpy-python-command'],
}
call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
os.chdir(workdir)
try:
call(bazel_command, env=env)
call(('tensorboard/pip_package/build_pip_package.sh',
os.path.join(workdir, 'dist')), env=env)
unpack_archive(os.path.join('dist', options['egg-name']), location)
finally:
for d in 'dist',:
if os.path.exists(d):
shutil.rmtree(d)
[versions]
tensorflow-tensorboard = 0.4.0
[buildout]
extends =
../../stack/slapos.cfg
../scipy/buildout.cfg
../zip/buildout.cfg
../bazel/buildout.cfg
../protobuf-python/buildout.cfg
../tensorboard/buildout.cfg
parts =
slapos-cookbook-develop
slapos-cookbook
tensorflow-build-install-egg
tensorflow
[numpy-egg]
recipe = zc.recipe.egg
......@@ -21,17 +16,22 @@ eggs =
${numpy:egg}
interpreter = numpy-python
scripts = numpy-python
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[tensorflow-repository]
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorflow
tag = v1.4.0
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
commit = v${versions:tensorflow}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-r1.4.patch ) || (rm -fr ${:location}; exit 1)
command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git reset --merge ${:commit}
git apply ${:_profile_base_location_}/tensorflow-r1.4.patch
) || { rm -fr ${:location}; exit 1; }
[cuda]
tf_need_cuda = 1
......@@ -41,9 +41,12 @@ tf_cuda_compute_capabilities = 5.2,6.1
cuda_toolkit_path = /usr/local/cuda
cudnn_install_path = /usr/local/cuda
[tensorflow-build]
[tensorflow]
recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${tensorflow-repository:location}
egg-name = ${:egg}-${versions:tensorflow}-py2.7-linux-x86_64.egg
location = ${buildout:eggs-directory}/${:egg-name}
gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64
......@@ -52,9 +55,12 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin
bazel_bin = ${bazel:location}/bin
need_cuda = ${cuda:cuda_toolkit_path}
depends =
${tensorflow_tensorboard:recipe}
${protobuf-python:egg}
install =
import os
os.makedirs(location)
import os, shutil
from setuptools.archive_util import unpack_archive
workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'],
......@@ -95,23 +101,16 @@ install =
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
or
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package'])
call(bazel_command, cwd=workdir, env=env)
os.chdir(workdir)
try:
call(bazel_command, env=env)
call(('bazel-bin/tensorflow/tools/pip_package/build_pip_package',
os.path.join(workdir, 'dist')))
unpack_archive(os.path.join('dist', options['egg-name']), location)
finally:
for d in 'dist',:
if os.path.exists(d):
shutil.rmtree(d)
[tensorflow-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorflow-repository-path = ${tensorflow-repository:location}
need-tensorboard-build = ${tensorboard-build:location}
need-tensorflow-build = ${tensorflow-build:location}
need-protobuf-python = ${protobuf-python:egg}
egg = tensorflow
install =
import os
os.makedirs(location)
workdir = options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions]
tensorflow = 1.4.0
......@@ -11,8 +11,8 @@ parts +=
[eggs]
eggs +=
${tensorboard-build-install-egg:egg}
${tensorflow-build-install-egg:egg}
${tensorflow_tensorboard:egg}
${tensorflow:egg}
${protobuf-python:egg}
${h5py:egg}
${pillow-python:egg}
......@@ -22,8 +22,8 @@ eggs +=
filelock
nose
${chainer:egg}
${pytorch-build-install-egg:egg}
${python-cocoapi-build-install-egg:egg}
${torch:egg}
${pycocotools:egg}
torchvision
[cuda]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment