Skip to content

slapos
slapos
Commit 0887b490 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

keras: some clean-up 

For repositories with patches, the latter should be downloaded first.

Another issue that remains unfixed is that during the build of some
components, some data are cached in ~/.cache and this can break further
builds with the same user.
parent 92fdea86
Hide whitespace changes
Inline Side-by-side
Showing with 110 additions and 179 deletions
component/keras/buildout.cfg
View file @ 0887b490
...@@ -14,7 +14,7 @@ initialization = ...@@ -14,7 +14,7 @@ initialization =
eggs = eggs =
${scipy:egg} ${scipy:egg}
${numpy:egg} ${numpy:egg}
${tensorflow-build-install-egg:egg} ${tensorflow:egg}
${protobuf-python:egg} ${protobuf-python:egg}
${python-PyYAML:egg} ${python-PyYAML:egg}
six six
...@@ -25,6 +25,10 @@ interpreter = keras-python ...@@ -25,6 +25,10 @@ interpreter = keras-python
scripts = keras-python scripts = keras-python
[versions] [versions]
backports.weakref = 1.0.post1
futures = 3.2.0
Keras = 2.1.0 Keras = 2.1.0
tensorflow = 1.4.0 markdown = 3.1.1
protobuf = 3.11.3
h5py = 2.7.0rc2 h5py = 2.7.0rc2
wheel = 0.33.6
component/python-cocoapi/buildout.cfg
View file @ 0887b490
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg
../gcc/buildout.cfg
../cython/buildout.cfg ../cython/buildout.cfg
../scipy/buildout.cfg ../scipy/buildout.cfg
parts = parts =
python-cocoapi-build-install-egg pycocotools
[python-cocoapi-repository] [python-cocoapi-repository]
recipe = plone.recipe.command recipe = slapos.recipe.build:gitclone
stop-on-error = true
repository = https://github.com/cocodataset/cocoapi.git repository = https://github.com/cocodataset/cocoapi.git
tag = master git-executable = ${git:location}/bin/git
git-binary = ${git:location}/bin/git revision = 0d86027d941482ae9649fb9e6e871704efc48b06
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/setup.py.patch ) || (rm -fr ${:location}; exit 1)
[python-cocoapi-build-interpreter] [pycocotools]
recipe = zc.recipe.egg recipe = zc.recipe.egg:develop
initialization = egg = ${:_buildout_section_name_}
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning setup = ${python-cocoapi-repository:location}/PythonAPI
eggs = environment = scipy-env
setuptools setup-eggs =
${cython:egg} ${cython:egg}
${scipy:egg}
${numpy:egg} ${numpy:egg}
interpreter = python-cocoapi-build-interpreter ${scipy:egg}
scripts = python-cocoapi-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[python-cocoapi-build]
recipe = slapos.recipe.build
workdir = ${python-cocoapi-repository:location}/PythonAPI
python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
gcc-location = ${gcc:location}
install =
import os
os.makedirs(location)
workdir = options['workdir']
python_bin = options['python-bin']
gcc_location = options['gcc-location']
env = {'PATH':':'.join([gcc_location+'/bin',
os.environ['PATH']]),
'CC':gcc_location+'/bin/gcc',
'CXX':gcc_location+'/bin/g++',
}
call([python_bin, 'setup.py', 'build_ext'], cwd=workdir, env=env)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
[python-cocoapi-build-install-egg]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
python-cocoapi-repository-path = ${python-cocoapi-repository:location}
need-python-cocoapi-build = ${python-cocoapi-build:location}
egg = pycocotools
install =
import os
os.makedirs(location)
workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
component/python-cocoapi/setup.py.patch deleted 100644 → 0
View file @ 92fdea86
diff --git a/PythonAPI/setup.py b/PythonAPI/setup.py
index eb3d508..d619b3c 100644
--- a/PythonAPI/setup.py
+++ b/PythonAPI/setup.py
@@ -1,4 +1,5 @@
-from distutils.core import setup
+#from distutils.core import setup
+from setuptools import setup
from Cython.Build import cythonize
from distutils.extension import Extension
import numpy as np
@@ -21,4 +22,4 @@ setup(name='pycocotools',
version='2.0',
ext_modules=
cythonize(ext_modules)
- )
\ No newline at end of file
+ )
component/pytorch/buildout.cfg
View file @ 0887b490
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg
../gcc/buildout.cfg ../gcc/buildout.cfg
../openblas/buildout.cfg ../openblas/buildout.cfg
../cmake/buildout.cfg ../cmake/buildout.cfg
...@@ -10,20 +9,25 @@ extends = ...@@ -10,20 +9,25 @@ extends =
../pillow/buildout.cfg ../pillow/buildout.cfg
../scipy/buildout.cfg ../scipy/buildout.cfg
../matplotlib/buildout.cfg ../matplotlib/buildout.cfg
../unzip/buildout.cfg
parts = parts =
pytorch-egg torch
[pytorch-repository] [pytorch-repository]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/pytorch/pytorch repository = https://github.com/pytorch/pytorch
tag = master
commit = 4af66c43045a317b477918c503d105f565b4a66b commit = 4af66c43045a317b477918c503d105f565b4a66b
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recursive --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:git-binary} checkout ${:commit}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/pytorch.4af66c4.patch ) || (rm -fr ${:location}; exit 1) command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git config submodule.recurse true
git reset --merge ${:commit}
git submodule update --init
git apply ${:_profile_base_location_}/pytorch.4af66c4.patch
) || { rm -fr ${:location}; exit 1; }
[pytorch-build-interpreter] [pytorch-build-interpreter]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -42,9 +46,12 @@ interpreter = pytorch-build-interpreter ...@@ -42,9 +46,12 @@ interpreter = pytorch-build-interpreter
scripts = pytorch-build-interpreter scripts = pytorch-build-interpreter
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pytorch-build] [torch]
recipe = slapos.recipe.build recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${pytorch-repository:location} workdir = ${pytorch-repository:location}
egg-name = ${:egg}-${versions:torch}-py2.7-linux-x86_64.egg
location = ${buildout:eggs-directory}/${:egg-name}
python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter} python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
no-cuda = 0 no-cuda = 0
cmake-bin = ${cmake:location}/bin cmake-bin = ${cmake:location}/bin
...@@ -53,8 +60,8 @@ binutils-location = ${binutils:location} ...@@ -53,8 +60,8 @@ binutils-location = ${binutils:location}
gcc-location = ${gcc:location} gcc-location = ${gcc:location}
openblas-location = ${openblas:location} openblas-location = ${openblas:location}
install = install =
import os import os, shutil
os.makedirs(location) from setuptools.archive_util import unpack_archive
workdir = options['workdir'] workdir = options['workdir']
python_bin = options['python-bin'] python_bin = options['python-bin']
binutils_location = options['binutils-location'] binutils_location = options['binutils-location']
...@@ -86,25 +93,15 @@ install = ...@@ -86,25 +93,15 @@ install =
} }
if options.get('no-cuda') == '1': if options.get('no-cuda') == '1':
env['NO_CUDA'] = '1' env['NO_CUDA'] = '1'
call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env) os.chdir(workdir)
call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env) try:
call([python_bin, 'setup.py', 'build'], env=env)
[pytorch-build-install-egg] call([python_bin, 'setup.py', 'bdist_egg'], env=env)
# BUG: This section does FS changes outside location. unpack_archive(os.path.join('dist', options['egg-name']), location)
recipe = slapos.recipe.build finally:
unzip-binary = ${unzip:location}/bin/unzip for d in 'build', 'dist':
pytorch-repository-path = ${pytorch-repository:location} if os.path.exists(d):
location = ${buildout:parts-directory}/${:_buildout_section_name_} shutil.rmtree(d)
need-pytorch-build = ${pytorch-build:location}
egg = torch
install =
import os
os.makedirs(location)
workdir = options['pytorch-repository-path']
egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[pytorch-egg] [pytorch-egg]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -114,9 +111,9 @@ eggs = ...@@ -114,9 +111,9 @@ eggs =
${scipy:egg} ${scipy:egg}
${numpy:egg} ${numpy:egg}
${python-PyYAML:egg} ${python-PyYAML:egg}
${pytorch-build-install-egg:egg} ${torch:egg}
${pillow-python:egg} ${pillow-python:egg}
${python-cocoapi-build-install-egg:egg} ${pycocotools:egg}
${matplotlib:egg} ${matplotlib:egg}
six six
torchvision torchvision
...@@ -125,3 +122,4 @@ scripts = pytorch-python ...@@ -125,3 +122,4 @@ scripts = pytorch-python
[versions] [versions]
torchvision = 0.1.6 torchvision = 0.1.6
torch = 0.2.0+4af66c4
component/tensorboard/buildout.cfg
View file @ 0887b490
[buildout] [buildout]
extends = extends =
../zip/buildout.cfg
../bazel/buildout.cfg ../bazel/buildout.cfg
../git/buildout.cfg
parts = parts =
slapos-cookbook-develop tensorflow_tensorboard
slapos-cookbook
tensorboard-build-install-egg
[tensorboard-repository] [tensorboard-repository]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorboard repository = https://github.com/tensorflow/tensorboard
tag = 0.4.0 commit = ${versions:tensorflow-tensorboard}
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1) command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git reset --merge ${:commit}
git apply ${:_profile_base_location_}/0.4.0.patch
) || { rm -fr ${:location}; exit 1; }
[tensorboard-build] [tensorflow_tensorboard]
recipe = slapos.recipe.build recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${tensorboard-repository:location} workdir = ${tensorboard-repository:location}
egg-name = ${:egg}-${versions:tensorflow-tensorboard}-py2.7.egg
location = ${buildout:eggs-directory}/${:egg-name}
gcc-bin = ${gcc:location}/bin gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64 gcc-lib64 = ${gcc:location}/lib64
...@@ -28,8 +34,8 @@ python27-lib = ${python2.7:location}/lib ...@@ -28,8 +34,8 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin java_home_bin = ${bazel:java_home}/bin
bazel-bin = ${bazel:location}/bin bazel-bin = ${bazel:location}/bin
install = install =
import os import os, shutil
os.makedirs(location) from setuptools.archive_util import unpack_archive
workdir = options['workdir'] workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'], env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'], options['java_home_bin'],
...@@ -45,30 +51,16 @@ install = ...@@ -45,30 +51,16 @@ install =
} }
env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH'] env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package'] bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
call(bazel_command, cwd=workdir, env=env) os.chdir(workdir)
try:
[tensorboard-build-install-egg] call(bazel_command, env=env)
recipe = slapos.recipe.build call(('tensorboard/pip_package/build_pip_package.sh',
unzip-binary = ${unzip:location}/bin/unzip os.path.join(workdir, 'dist')), env=env)
tensorboard-repository-path = ${tensorboard-repository:location} unpack_archive(os.path.join('dist', options['egg-name']), location)
need-tensorboard-build = ${tensorboard-build:location} finally:
egg = tensorflow-tensorboard for d in 'dist',:
bazel-bin = ${bazel:location}/bin if os.path.exists(d):
java_home_bin = ${bazel:java_home}/bin shutil.rmtree(d)
numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
install =
import os
os.makedirs(location)
workdir = options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([options['bazel-bin'],
options['java_home_bin'],
os.environ['PATH']]),
'PYTHON_BIN_PATH':options['numpy-python-command'],
}
call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
[versions] [versions]
tensorflow-tensorboard = 0.4.0
component/tensorflow/buildout.cfg
View file @ 0887b490
[buildout] [buildout]
extends = extends =
../../stack/slapos.cfg
../scipy/buildout.cfg ../scipy/buildout.cfg
../zip/buildout.cfg
../bazel/buildout.cfg
../protobuf-python/buildout.cfg ../protobuf-python/buildout.cfg
../tensorboard/buildout.cfg ../tensorboard/buildout.cfg
parts = parts =
slapos-cookbook-develop tensorflow
slapos-cookbook
tensorflow-build-install-egg
[numpy-egg] [numpy-egg]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -21,17 +16,22 @@ eggs = ...@@ -21,17 +16,22 @@ eggs =
${numpy:egg} ${numpy:egg}
interpreter = numpy-python interpreter = numpy-python
scripts = numpy-python scripts = numpy-python
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[tensorflow-repository] [tensorflow-repository]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
repository = https://github.com/tensorflow/tensorflow repository = https://github.com/tensorflow/tensorflow
tag = v1.4.0 commit = v${versions:tensorflow}
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-r1.4.patch ) || (rm -fr ${:location}; exit 1) command =
set -e
export PATH=${git:location}/bin:$PATH
( git clone --quiet -n ${:repository} ${:location}
cd ${:location}
git reset --merge ${:commit}
git apply ${:_profile_base_location_}/tensorflow-r1.4.patch
) || { rm -fr ${:location}; exit 1; }
[cuda] [cuda]
tf_need_cuda = 1 tf_need_cuda = 1
...@@ -41,9 +41,12 @@ tf_cuda_compute_capabilities = 5.2,6.1 ...@@ -41,9 +41,12 @@ tf_cuda_compute_capabilities = 5.2,6.1
cuda_toolkit_path = /usr/local/cuda cuda_toolkit_path = /usr/local/cuda
cudnn_install_path = /usr/local/cuda cudnn_install_path = /usr/local/cuda
[tensorflow-build] [tensorflow]
recipe = slapos.recipe.build recipe = slapos.recipe.build
egg = ${:_buildout_section_name_}
workdir = ${tensorflow-repository:location} workdir = ${tensorflow-repository:location}
egg-name = ${:egg}-${versions:tensorflow}-py2.7-linux-x86_64.egg
location = ${buildout:eggs-directory}/${:egg-name}
gcc-bin = ${gcc:location}/bin gcc-bin = ${gcc:location}/bin
gcc-lib = ${gcc:location}/lib gcc-lib = ${gcc:location}/lib
gcc-lib64 = ${gcc:location}/lib64 gcc-lib64 = ${gcc:location}/lib64
...@@ -52,9 +55,12 @@ python27-lib = ${python2.7:location}/lib ...@@ -52,9 +55,12 @@ python27-lib = ${python2.7:location}/lib
java_home_bin = ${bazel:java_home}/bin java_home_bin = ${bazel:java_home}/bin
bazel_bin = ${bazel:location}/bin bazel_bin = ${bazel:location}/bin
need_cuda = ${cuda:cuda_toolkit_path} need_cuda = ${cuda:cuda_toolkit_path}
depends =
${tensorflow_tensorboard:recipe}
${protobuf-python:egg}
install = install =
import os import os, shutil
os.makedirs(location) from setuptools.archive_util import unpack_archive
workdir = options['workdir'] workdir = options['workdir']
env = {'PATH':':'.join([options['gcc-bin'], env = {'PATH':':'.join([options['gcc-bin'],
options['java_home_bin'], options['java_home_bin'],
...@@ -95,23 +101,16 @@ install = ...@@ -95,23 +101,16 @@ install =
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package'] ['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
or or
['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']) ['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package'])
call(bazel_command, cwd=workdir, env=env) os.chdir(workdir)
try:
call(bazel_command, env=env)
call(('bazel-bin/tensorflow/tools/pip_package/build_pip_package',
os.path.join(workdir, 'dist')))
unpack_archive(os.path.join('dist', options['egg-name']), location)
finally:
for d in 'dist',:
if os.path.exists(d):
shutil.rmtree(d)
[tensorflow-build-install-egg] [versions]
# BUG: This section does FS changes outside location. tensorflow = 1.4.0
recipe = slapos.recipe.build
unzip-binary = ${unzip:location}/bin/unzip
tensorflow-repository-path = ${tensorflow-repository:location}
need-tensorboard-build = ${tensorboard-build:location}
need-tensorflow-build = ${tensorflow-build:location}
need-protobuf-python = ${protobuf-python:egg}
egg = tensorflow
install =
import os
os.makedirs(location)
workdir = options['tensorflow-repository-path']
egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
software/wendelin/software-kerastensorflow.cfg
View file @ 0887b490
...@@ -11,8 +11,8 @@ parts += ...@@ -11,8 +11,8 @@ parts +=
[eggs] [eggs]
eggs += eggs +=
${tensorboard-build-install-egg:egg} ${tensorflow_tensorboard:egg}
${tensorflow-build-install-egg:egg} ${tensorflow:egg}
${protobuf-python:egg} ${protobuf-python:egg}
${h5py:egg} ${h5py:egg}
${pillow-python:egg} ${pillow-python:egg}
...@@ -22,8 +22,8 @@ eggs += ...@@ -22,8 +22,8 @@ eggs +=
filelock filelock
nose nose
${chainer:egg} ${chainer:egg}
${pytorch-build-install-egg:egg} ${torch:egg}
${python-cocoapi-build-install-egg:egg} ${pycocotools:egg}
torchvision torchvision
[cuda] [cuda]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7