keras: some clean-up

For repositories with patches, the latter should be downloaded first.

Another issue that remains unfixed is that during the build of some
components, some data are cached in ~/.cache and this can break further
builds with the same user.

component/keras/buildout.cfg

@@ -14,7 +14,7 @@ initialization =
 eggs = 
   ${scipy:egg}
   ${numpy:egg}
-  ${tensorflow-build-install-egg:egg}
+  ${tensorflow:egg}
   ${protobuf-python:egg}
   ${python-PyYAML:egg}
   six
@@ -25,6 +25,10 @@ interpreter = keras-python
 scripts = keras-python
 
 [versions]
+backports.weakref = 1.0.post1
+futures = 3.2.0
 Keras = 2.1.0
-tensorflow = 1.4.0
+markdown = 3.1.1
+protobuf = 3.11.3
 h5py = 2.7.0rc2
+wheel = 0.33.6

component/python-cocoapi/buildout.cfg

 [buildout]
 extends =
-  ../../stack/slapos.cfg
-  ../gcc/buildout.cfg
   ../cython/buildout.cfg
   ../scipy/buildout.cfg
 parts =
-  python-cocoapi-build-install-egg
+  pycocotools
 
 [python-cocoapi-repository]
-recipe = plone.recipe.command
-stop-on-error = true
+recipe = slapos.recipe.build:gitclone
 repository = https://github.com/cocodataset/cocoapi.git
-tag = master
-git-binary = ${git:location}/bin/git
-patch-binary = ${patch:location}/bin/patch
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/setup.py.patch ) || (rm -fr ${:location}; exit 1)
+git-executable = ${git:location}/bin/git
+revision = 0d86027d941482ae9649fb9e6e871704efc48b06
 
-[python-cocoapi-build-interpreter]
-recipe = zc.recipe.egg
-initialization =
-  import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
-eggs =
-  setuptools
+[pycocotools]
+recipe = zc.recipe.egg:develop
+egg = ${:_buildout_section_name_}
+setup = ${python-cocoapi-repository:location}/PythonAPI
+environment = scipy-env
+setup-eggs =
   ${cython:egg}
-  ${scipy:egg}
   ${numpy:egg}
-interpreter = python-cocoapi-build-interpreter
-scripts = python-cocoapi-build-interpreter
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-
-[python-cocoapi-build]
-recipe = slapos.recipe.build
-workdir = ${python-cocoapi-repository:location}/PythonAPI
-python-bin = ${buildout:bin-directory}/${python-cocoapi-build-interpreter:interpreter}
-gcc-location = ${gcc:location}
-install =
-  import os
-  os.makedirs(location)
-  workdir = options['workdir']
-  python_bin = options['python-bin']
-  gcc_location = options['gcc-location']
-  env = {'PATH':':'.join([gcc_location+'/bin',
-                          os.environ['PATH']]),
-         'CC':gcc_location+'/bin/gcc',
-         'CXX':gcc_location+'/bin/g++',
-         }
-  call([python_bin, 'setup.py', 'build_ext'], cwd=workdir, env=env)
-  call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
-
-[python-cocoapi-build-install-egg]
-# BUG: This section does FS changes outside location.
-recipe = slapos.recipe.build
-unzip-binary = ${unzip:location}/bin/unzip
-python-cocoapi-repository-path = ${python-cocoapi-repository:location}
-need-python-cocoapi-build = ${python-cocoapi-build:location}
-egg = pycocotools
-install =
-  import os
-  os.makedirs(location)
-  workdir = options['python-cocoapi-repository-path']+'/PythonAPI'
-  egg_name = 'pycocotools-2.0-py2.7-linux-x86_64.egg'
-  dist_dir = os.path.join(workdir, 'dist')
-  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  ${scipy:egg}

component/python-cocoapi/setup.py.patch

-diff --git a/PythonAPI/setup.py b/PythonAPI/setup.py
-index eb3d508..d619b3c 100644
---- a/PythonAPI/setup.py
-+++ b/PythonAPI/setup.py
-@@ -1,4 +1,5 @@
--from distutils.core import setup
-+#from distutils.core import setup
-+from setuptools import setup
- from Cython.Build import cythonize
- from distutils.extension import Extension
- import numpy as np
-@@ -21,4 +22,4 @@ setup(name='pycocotools',
-       version='2.0',
-       ext_modules=
-           cythonize(ext_modules)
--      )
-\ No newline at end of file
-+      )

component/pytorch/buildout.cfg

 [buildout]
 extends =
-  ../../stack/slapos.cfg
   ../gcc/buildout.cfg
   ../openblas/buildout.cfg
   ../cmake/buildout.cfg
@@ -10,20 +9,25 @@ extends =
   ../pillow/buildout.cfg
   ../scipy/buildout.cfg
   ../matplotlib/buildout.cfg
-  ../unzip/buildout.cfg
 parts =
-  pytorch-egg
+  torch
 
 [pytorch-repository]
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/pytorch/pytorch
-tag = master
 commit = 4af66c43045a317b477918c503d105f565b4a66b
-git-binary = ${git:location}/bin/git
-patch-binary = ${patch:location}/bin/patch
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --recursive --quiet -b ${:tag} ${:repository} ${:location}; cd ${:location}; ${:git-binary} checkout ${:commit}; ${:patch-binary} -p1 -d . < ${:_profile_base_location_}/pytorch.4af66c4.patch ) || (rm -fr ${:location}; exit 1)
+command =
+  set -e
+  export PATH=${git:location}/bin:$PATH
+  ( git clone --quiet -n ${:repository} ${:location}
+    cd ${:location}
+    git config submodule.recurse true
+    git reset --merge ${:commit}
+    git submodule update --init
+    git apply ${:_profile_base_location_}/pytorch.4af66c4.patch
+  ) || { rm -fr ${:location}; exit 1; }
 
 [pytorch-build-interpreter]
 recipe = zc.recipe.egg
@@ -42,9 +46,12 @@ interpreter = pytorch-build-interpreter
 scripts = pytorch-build-interpreter
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
-[pytorch-build]
+[torch]
 recipe = slapos.recipe.build
+egg = ${:_buildout_section_name_}
 workdir = ${pytorch-repository:location}
+egg-name = ${:egg}-${versions:torch}-py2.7-linux-x86_64.egg
+location = ${buildout:eggs-directory}/${:egg-name}
 python-bin = ${buildout:bin-directory}/${pytorch-build-interpreter:interpreter}
 no-cuda = 0
 cmake-bin = ${cmake:location}/bin
@@ -53,8 +60,8 @@ binutils-location = ${binutils:location}
 gcc-location = ${gcc:location}
 openblas-location = ${openblas:location}
 install =
-  import os
-  os.makedirs(location)
+  import os, shutil
+  from setuptools.archive_util import unpack_archive
   workdir = options['workdir']
   python_bin = options['python-bin']
   binutils_location = options['binutils-location']
@@ -86,25 +93,15 @@ install =
          }
   if options.get('no-cuda') == '1':
     env['NO_CUDA'] = '1'
-  call([python_bin, 'setup.py', 'build'], cwd=workdir, env=env)
-  call([python_bin, 'setup.py', 'bdist_egg'], cwd=workdir, env=env)
-
-[pytorch-build-install-egg]
-# BUG: This section does FS changes outside location.
-recipe = slapos.recipe.build
-unzip-binary = ${unzip:location}/bin/unzip
-pytorch-repository-path = ${pytorch-repository:location}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-need-pytorch-build = ${pytorch-build:location}
-egg = torch
-install =
-  import os
-  os.makedirs(location)
-  workdir = options['pytorch-repository-path']
-  egg_name = 'torch-0.2.0+4af66c4-py2.7-linux-x86_64.egg'
-  dist_dir = os.path.join(workdir, 'dist')
-  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  os.chdir(workdir)
+  try:
+    call([python_bin, 'setup.py', 'build'], env=env)
+    call([python_bin, 'setup.py', 'bdist_egg'], env=env)
+    unpack_archive(os.path.join('dist', options['egg-name']), location)
+  finally:
+    for d in 'build', 'dist':
+      if os.path.exists(d):
+        shutil.rmtree(d)
 
 [pytorch-egg]
 recipe = zc.recipe.egg
@@ -114,9 +111,9 @@ eggs =
   ${scipy:egg}
   ${numpy:egg}
   ${python-PyYAML:egg}
-  ${pytorch-build-install-egg:egg}
+  ${torch:egg}
   ${pillow-python:egg}
-  ${python-cocoapi-build-install-egg:egg}
+  ${pycocotools:egg}
   ${matplotlib:egg}
   six
   torchvision
@@ -125,3 +122,4 @@ scripts = pytorch-python
 
 [versions]
 torchvision = 0.1.6
+torch = 0.2.0+4af66c4

component/tensorboard/buildout.cfg

 [buildout]
 extends =
-  ../zip/buildout.cfg
   ../bazel/buildout.cfg
+  ../git/buildout.cfg
 parts =
-  slapos-cookbook-develop
-  slapos-cookbook
-  tensorboard-build-install-egg
+  tensorflow_tensorboard
 
 [tensorboard-repository]
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorboard
-tag = 0.4.0
-git-binary = ${git:location}/bin/git
-patch-binary = ${patch:location}/bin/patch
+commit = ${versions:tensorflow-tensorboard}
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
+command =
+  set -e
+  export PATH=${git:location}/bin:$PATH
+  ( git clone --quiet -n ${:repository} ${:location}
+    cd ${:location}
+    git reset --merge ${:commit}
+    git apply ${:_profile_base_location_}/0.4.0.patch
+  ) || { rm -fr ${:location}; exit 1; }
 
-[tensorboard-build]
+[tensorflow_tensorboard]
 recipe = slapos.recipe.build
+egg = ${:_buildout_section_name_}
 workdir = ${tensorboard-repository:location}
+egg-name = ${:egg}-${versions:tensorflow-tensorboard}-py2.7.egg
+location = ${buildout:eggs-directory}/${:egg-name}
 gcc-bin = ${gcc:location}/bin
 gcc-lib = ${gcc:location}/lib
 gcc-lib64 = ${gcc:location}/lib64
@@ -28,8 +34,8 @@ python27-lib = ${python2.7:location}/lib
 java_home_bin = ${bazel:java_home}/bin
 bazel-bin = ${bazel:location}/bin
 install =
-  import os
-  os.makedirs(location)
+  import os, shutil
+  from setuptools.archive_util import unpack_archive
   workdir = options['workdir']
   env = {'PATH':':'.join([options['gcc-bin'],
                           options['java_home_bin'],
@@ -45,30 +51,16 @@ install =
          }
   env['LD_LIBRARY_PATH'] = env['LIBRARY_PATH']
   bazel_command = ['bazel', 'build', '--spawn_strategy=standalone', '--verbose_failures', '--sandbox_debug', '//tensorboard/pip_package:build_pip_package']
-  call(bazel_command, cwd=workdir, env=env)
-
-[tensorboard-build-install-egg]
-recipe = slapos.recipe.build
-unzip-binary = ${unzip:location}/bin/unzip
-tensorboard-repository-path = ${tensorboard-repository:location}
-need-tensorboard-build = ${tensorboard-build:location}
-egg = tensorflow-tensorboard
-bazel-bin = ${bazel:location}/bin
-java_home_bin = ${bazel:java_home}/bin
-numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
-install =
-  import os
-  os.makedirs(location)
-  workdir = options['tensorboard-repository-path']
-  egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
-  dist_dir = os.path.join(workdir, 'dist')
-  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  env = {'PATH':':'.join([options['bazel-bin'],
-                          options['java_home_bin'],
-                          os.environ['PATH']]),
-         'PYTHON_BIN_PATH':options['numpy-python-command'],
-         }
-  call(['tensorboard/pip_package/build_pip_package.sh', dist_dir], cwd=workdir, env=env)
-  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+  os.chdir(workdir)
+  try:
+    call(bazel_command, env=env)
+    call(('tensorboard/pip_package/build_pip_package.sh',
+          os.path.join(workdir, 'dist')), env=env)
+    unpack_archive(os.path.join('dist', options['egg-name']), location)
+  finally:
+    for d in 'dist',:
+      if os.path.exists(d):
+        shutil.rmtree(d)
 
 [versions]
+tensorflow-tensorboard = 0.4.0

component/tensorflow/buildout.cfg

 [buildout]
 extends =
-  ../../stack/slapos.cfg
   ../scipy/buildout.cfg
-  ../zip/buildout.cfg
-  ../bazel/buildout.cfg
   ../protobuf-python/buildout.cfg
   ../tensorboard/buildout.cfg
 parts =
-  slapos-cookbook-develop
-  slapos-cookbook
-  tensorflow-build-install-egg
+  tensorflow
 
 [numpy-egg]
 recipe = zc.recipe.egg
@@ -21,17 +16,22 @@ eggs =
   ${numpy:egg}
 interpreter = numpy-python
 scripts = numpy-python
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [tensorflow-repository]
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorflow
-tag = v1.4.0
-git-binary = ${git:location}/bin/git
-patch-binary = ${patch:location}/bin/patch
+commit = v${versions:tensorflow}
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --recurse-submodules --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/tensorflow-r1.4.patch ) || (rm -fr ${:location}; exit 1)
+command =
+  set -e
+  export PATH=${git:location}/bin:$PATH
+  ( git clone --quiet -n ${:repository} ${:location}
+    cd ${:location}
+    git reset --merge ${:commit}
+    git apply ${:_profile_base_location_}/tensorflow-r1.4.patch
+  ) || { rm -fr ${:location}; exit 1; }
+
 
 [cuda]
 tf_need_cuda = 1
@@ -41,9 +41,12 @@ tf_cuda_compute_capabilities = 5.2,6.1
 cuda_toolkit_path = /usr/local/cuda
 cudnn_install_path = /usr/local/cuda
 
-[tensorflow-build]
+[tensorflow]
 recipe = slapos.recipe.build
+egg = ${:_buildout_section_name_}
 workdir = ${tensorflow-repository:location}
+egg-name = ${:egg}-${versions:tensorflow}-py2.7-linux-x86_64.egg
+location = ${buildout:eggs-directory}/${:egg-name}
 gcc-bin = ${gcc:location}/bin
 gcc-lib = ${gcc:location}/lib
 gcc-lib64 = ${gcc:location}/lib64
@@ -52,9 +55,12 @@ python27-lib = ${python2.7:location}/lib
 java_home_bin = ${bazel:java_home}/bin
 bazel_bin = ${bazel:location}/bin
 need_cuda = ${cuda:cuda_toolkit_path}
+depends =
+  ${tensorflow_tensorboard:recipe}
+  ${protobuf-python:egg}
 install =
-  import os
-  os.makedirs(location)
+  import os, shutil
+  from setuptools.archive_util import unpack_archive
   workdir = options['workdir']
   env = {'PATH':':'.join([options['gcc-bin'],
                           options['java_home_bin'],
@@ -95,23 +101,16 @@ install =
     ['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '--config', 'cuda', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package']
     or
     ['bazel', 'build', '-c', 'opt', '--copt', '-march=native', '-s', '--verbose_failures', '//tensorflow/tools/pip_package:build_pip_package'])
-  call(bazel_command, cwd=workdir, env=env)
+  os.chdir(workdir)
+  try:
+    call(bazel_command, env=env)
+    call(('bazel-bin/tensorflow/tools/pip_package/build_pip_package',
+          os.path.join(workdir, 'dist')))
+    unpack_archive(os.path.join('dist', options['egg-name']), location)
+  finally:
+    for d in 'dist',:
+      if os.path.exists(d):
+        shutil.rmtree(d)
 
-[tensorflow-build-install-egg]
-# BUG: This section does FS changes outside location.
-recipe = slapos.recipe.build
-unzip-binary = ${unzip:location}/bin/unzip
-tensorflow-repository-path = ${tensorflow-repository:location}
-need-tensorboard-build = ${tensorboard-build:location}
-need-tensorflow-build = ${tensorflow-build:location}
-need-protobuf-python = ${protobuf-python:egg}
-egg = tensorflow
-install =
-  import os
-  os.makedirs(location)
-  workdir = options['tensorflow-repository-path']
-  egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
-  dist_dir = os.path.join(workdir, 'dist')
-  dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
-  call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)
-  call([options['unzip-binary'], '-o', os.path.join(dist_dir, egg_name), '-d', dest_dir])
+[versions]
+tensorflow = 1.4.0

software/wendelin/software-kerastensorflow.cfg

@@ -11,8 +11,8 @@ parts +=
 
 [eggs]
 eggs +=
-  ${tensorboard-build-install-egg:egg}
-  ${tensorflow-build-install-egg:egg}
+  ${tensorflow_tensorboard:egg}
+  ${tensorflow:egg}
   ${protobuf-python:egg}
   ${h5py:egg}
   ${pillow-python:egg}
@@ -22,8 +22,8 @@ eggs +=
   filelock
   nose
   ${chainer:egg}
-  ${pytorch-build-install-egg:egg}
-  ${python-cocoapi-build-install-egg:egg}
+  ${torch:egg}
+  ${pycocotools:egg}
   torchvision
 
 [cuda]