obs: Integrate new bootstraping method to generic.

Integrate to generic the new bootstraping method from ```obs/slapos```.
Update some egg versions.

obs/_generic/build-scripts/00env.sh

@@ -24,7 +24,7 @@ TARGET_DIR=$(realpath -m "$TARGET_DIR")
 # Source package
 PACKAGE_SECTION=${PACKAGE_SECTION:-utils}
 PACKAGE_PRIORITY=${PACKAGE_PRIORITY:-optional}
-PACKAGE_BUILD_DEPENDENCIES=${PACKAGE_BUILD_DEPENDENCIES:-debhelper (>= 4.1.16), chrpath, python (>= 2.7),}
+PACKAGE_BUILD_DEPENDENCIES=${PACKAGE_BUILD_DEPENDENCIES:-debhelper, chrpath, python3 (>= 3.7) | python, python3-dev (>= 3.7) | python-dev}
 # Binary package
 PACKAGE_ARCHITECTURE=${PACKAGE_ARCHITECTURE:-any}
 PACKAGE_DEPENDENCIES=${PACKAGE_DEPENDENCIES:-\$\{misc:Depends\}, \$\{shlibs:Depends\},}
@@ -37,9 +37,6 @@ PACKAGE_URGENCY=${PACKAGE_URGENCY:-medium}
 GIT_REPOSITORY=${GIT_REPOSITORY:-https://lab.nexedi.com/nexedi/slapos}
 GIT_BRANCH_OR_COMMIT=${GIT_BRANCH_OR_COMMIT:-master}
 BUILDOUT_RELATIVE_PATH=${BUILDOUT_RELATIVE_PATH:-software/$SOFTWARE_NAME/software.cfg}
-## Versions
-SETUPTOOLS_VERSION=${SETUPTOOLS_VERSION:-63.2.0}
-ZC_BUILDOUT_VERSION=${ZC_BUILDOUT_VERSION:-2.7.1+slapos019}
 ## OBS information
 OBS_PROJECT=${OBS_PROJECT:-home:VIFIBnexedi:branches:home:VIFIBnexedi}
 OBS_PACKAGE=${OBS_PACKAGE:-$SOFTWARE_NAME}
@@ -55,6 +52,13 @@ COMPILATION_FILES_GENERIC_DIR="$INITIAL_DIR/_generic/compilation"
 COMPILATION_FILES_SOFTWARE_DIR="$INITIAL_DIR/$SOFTWARE_NAME/compilation"
 BUILD_DIR="$TARBALL_DIR/build"
 RUN_BUILDOUT_DIR="$BUILD_DIR/$TARGET_DIR"
+## Versions
+# OLD_SETUPTOOLS_VERSION must be strictly older than SETUPTOOLS_VERSION
+OLD_SETUPTOOLS_VERSION=44.1.0
+SETUPTOOLS_VERSION=44.1.1
+ZC_BUILDOUT_VERSION=2.7.1+slapos019
+ZC_RECIPE_EGG_VERSION=2.0.3+slapos003
+SLAPOS_LIBNETWORKCACHE_VERSION=0.25
 DISTRIB_FILES_GENERIC_DIR="$INITIAL_DIR/_generic/distributions"
 DISTRIB_FILES_SOFTWARE_DIR="$INITIAL_DIR/$SOFTWARE_NAME/distributions"
 BUILDOUT_DIR="$TARBALL_DIR/slapos_repository"

obs/_generic/build-scripts/10tarball_directory.sh

@@ -26,25 +26,49 @@ mkdir -p "$RUN_BUILDOUT_DIR"/{eggs,extends-cache,download-cache/dist}
 # with the one from <software_name>
 copy_and_solve_templates "$COMPILATION_FILES_GENERIC_DIR" "$TARBALL_DIR"
 copy_and_solve_templates "$COMPILATION_FILES_SOFTWARE_DIR" "$TARBALL_DIR"
-mv "$TARBALL_DIR/local_buildout.cfg" "$RUN_BUILDOUT_DIR/buildout.cfg"
 
 # BUILDOUT: BOOTSTRAPING AND RUN
 
 ### Download the bootstrap script
 mkdir -p "$RUN_BUILDOUT_DIR"
 cd "$RUN_BUILDOUT_DIR"
-wget https://bootstrap.pypa.io/bootstrap-buildout.py
+wget https://lab.nexedi.com/nexedi/slapos.buildout/raw/master/bootstrap/bootstrap.py
 
 ### Run buildout
 # Note: it creates a lot of things in $RUN_BUILDOUT_DIR/eggs/ and uses software_release/ at some point
 
-# bootstrap buildout (creates $RUN_BUILDOUT_DIR/bin/buildout)
-python2.7 -S bootstrap-buildout.py \
-	--buildout-version "$ZC_BUILDOUT_VERSION" \
-	--setuptools-version "$SETUPTOOLS_VERSION" \
-	--setuptools-to-dir eggs -f http://www.nexedi.org/static/packages/source/slapos.buildout/
-# run $RUN_BUILDOUT_DIR/bin/buildout (note that it modifies itself via rebootstrapping when compiling python)
-./bin/buildout -v
+# 00) Bootstrap buildout with an old version.
+#    no --buildout-version option, so it uses an upstream version of buildout
+#    creates $RUN_BUILDOUT_DIR/bin/buildout
+echo "[buildout]" > buildout.cfg # dummy .cfg file only for boostraping
+echo "Bootsrapping buildout..."
+python3 -S bootstrap.py \
+	--setuptools-version 44.1.0 \
+	--setuptools-to-dir eggs
+
+# 10) Get newest version of zc.buildout and setuptools.
+#    note that we can't directly do setuptools + zc.buildout +
+#    slapos.libnetworkcache because buildout would be relaunched in the middle
+#    without the "-S" option to python
+echo "Downloading the desired version of setuptools and zc.buildout..."
+cp "$TARBALL_DIR"/10cache-zc-buildout.cfg buildout.cfg
+sed -i '1s/$/ -S/' bin/buildout
+sed -i "/def _satisfied(/s/\(\bsource=\)None/\11/" eggs/zc.buildout-*/zc/buildout/easy_install.py # no wheel
+bin/buildout buildout:newest=true -v
+ls download-cache/dist/*.whl && { echo "There shouldn't be any wheel in download-cache" ; exit 1 ; }
+
+# 20) Compile a very simple buildout with networkcache.
+echo "Preparing networkcached zc.buildout..."
+cp "$TARBALL_DIR"/20cache-and-use-libnetworkcache.cfg buildout.cfg
+sed -i '1s/$/ -S/' bin/buildout
+bin/buildout buildout:newest=true -v
+
+# 30) Run $RUN_BUILDOUT_DIR/bin/buildout.
+#    note that it modifies itself via rebootstrapping when compiling python
+#    build locally everything with gcc to get download-cache and extends-cache ready
+echo "Finally running buildout for the local compilation..."
+cp "$TARBALL_DIR"/30local_buildout.cfg buildout.cfg
+bin/buildout buildout:newest=true -v
 
 ### Fix the go/ directory.
 

obs/_generic/build-scripts/20obs.sh

@@ -15,7 +15,7 @@ cd "$INITIAL_DIR"
 # copy compilation files and override the files from _generic
 # with the one from <software_name>
 copy_and_solve_templates "$DISTRIB_FILES_GENERIC_DIR" "$OBS_DIR"
-mv "$OBS_DIR"/_generic.dsc "$OBS_DIR/$SOFTWARE_AND_VERSION.dsc"
+mv "$OBS_DIR"/_generic.dsc "$OBS_DIR/$SOFTWARE_NAME.dsc"
 mv "$OBS_DIR"/_generic.spec "$OBS_DIR/$SOFTWARE_NAME.spec"
 copy_and_solve_templates "$DISTRIB_FILES_SOFTWARE_DIR" "$OBS_DIR"
 
@@ -23,14 +23,18 @@ copy_and_solve_templates "$DISTRIB_FILES_SOFTWARE_DIR" "$OBS_DIR"
 
 ### Finalize the tarball directory preparation
 # switch to the buildout wrapper for OBS
-cp "$TARBALL_DIR/obs_buildout.cfg" "$RUN_BUILDOUT_DIR/buildout.cfg"
+cp "$TARBALL_DIR"/40obs_buildout.cfg "$RUN_BUILDOUT_DIR"/buildout.cfg
 # save the local $TARBALL_DIR path so that it is replaced by the $TARBALL_DIR path from OBS' VM
 echo "$TARBALL_DIR" > "$TARBALL_DIR"/local_tarball_directory_path
 # add a stamp so that OBS does not clean the local preparation before compiling
 touch "$TARBALL_DIR/clean-stamp"
+# copy required eggs that are not cached
 # clean the compilation related files
 rm -rf "$RUN_BUILDOUT_DIR"/{.installed.cfg,downloads,parts,eggs,develop-eggs,bin,rebootstrap}
 
+rm -rf "$BUILDOUT_DIR"/.git # lighten what is sent to OBS
+# TODO: support extends-cache
+
 ### Prepare the archives for OBS
 # -C option allows to give tar an absolute path without archiving the directory from / (i.e. home/user/[...])
 tar czf "$OBS_DIR/$SOFTWARE_AND_VERSION$ARCHIVE_EXT" -C "$INITIAL_DIR/_tarballs" "$SOFTWARE_AND_VERSION"

obs/_generic/compilation/10cache-zc-buildout.cfg.in

+[buildout]
+rootdir = %TARGET_DIRECTORY%
+destdir = %BUILD_ROOT_DIRECTORY%
+builddir = %BUILD_DIRECTORY%
+extensions =
+extends-cache = extends-cache
+download-cache = download-cache
+parts =
+  zc.buildout
+
+# Add location for modified non-official slapos.buildout
+find-links +=
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+[zc.buildout]
+recipe = zc.recipe.egg
+eggs =
+  zc.buildout
+
+[versions]
+setuptools = %SETUPTOOLS_VERSION%
+zc.buildout = %ZC_BUILDOUT_VERSION%
+zc.recipe.egg = %ZC_RECIPE_EGG_VERSION%

obs/_generic/compilation/20cache-and-use-libnetworkcache.cfg.in

+[buildout]
+rootdir = %TARGET_DIRECTORY%
+destdir = %BUILD_ROOT_DIRECTORY%
+builddir = %BUILD_DIRECTORY%
+extensions =
+extends-cache = extends-cache
+download-cache = download-cache
+parts =
+  networkcached
+
+# Add location for modified non-official slapos.buildout
+find-links +=
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+[networkcached]
+recipe = zc.recipe.egg
+eggs =
+  slapos.libnetworkcache
+  zc.buildout
+
+[versions]
+setuptools = %SETUPTOOLS_VERSION%
+zc.buildout = %ZC_BUILDOUT_VERSION%
+slapos.libnetworkcache = %SLAPOS_LIBNETWORKCACHE%
+zc.recipe.egg = %ZC_RECIPE_EGG_VERSION%

obs/_generic/compilation/obs_buildout.cfg.in → obs/_generic/compilation/30local_buildout.cfg.in

@@ -11,3 +11,4 @@ download-cache = download-cache
 [versions]
 setuptools = %SETUPTOOLS_VERSION%
 zc.buildout = %ZC_BUILDOUT_VERSION%
+zc.recipe.egg = %ZC_RECIPE_EGG_VERSION%

obs/fluent-bit/env.sh

@@ -8,6 +8,5 @@ export SOFTWARE_NAME=fluent-bit
 export SOFTWARE_VERSION=1.9.7+1.0.277
 export GIT_BRANCH_OR_COMMIT=1.0.277
 
-export PACKAGE_BUILD_DEPENDENCIES="debhelper, chrpath, python3 (>=3.7) | python, python3-dev (>= 3.7) | python-dev"
 export SETUPTOOLS_VERSION=44.1.1
 export ZC_BUILDOUT_VERSION=2.7.1+slapos019

obs/lunzip/compilation/local_buildout.cfg.in

-[buildout]
-rootdir = %TARGET_DIR%
-destdir = %BUILD_DIR%
-builddir = %RUN_BUILDOUT_DIR%
-extends =
-  %BUILDOUT_ENTRY_POINT%
-versions = versions
-extends-cache = extends-cache
-download-cache = download-cache
-
-[versions]
-setuptools = %SETUPTOOLS_VERSION%
-zc.buildout = %ZC_BUILDOUT_VERSION%

obs/lunzip/compilation/obs_buildout.cfg.in

-[buildout]
-rootdir = %TARGET_DIR%
-destdir = %BUILD_DIR%
-builddir = %RUN_BUILDOUT_DIR%
-extends =
-  %BUILDOUT_ENTRY_POINT%
-versions = versions
-extends-cache = extends-cache
-download-cache = download-cache
-
-[versions]
-setuptools = %SETUPTOOLS_VERSION%
-zc.buildout = %ZC_BUILDOUT_VERSION%

obs/lunzip/env.sh

@@ -8,5 +8,5 @@ export MAINTAINER_EMAIL="ophelie.gagnard@nexedi.com"
 export TARGET_DIR="/"
 export BUILDOUT_RELATIVE_PATH=component/lunzip/buildout.cfg
 export SETUPTOOLS_VERSION=44.1.1
-export ZC_BUILDOUT_VERSION=2.7.1+slapos016
+export ZC_BUILDOUT_VERSION=2.7.1+slapos019
 export OBS_PROJECT=home:oph.nxd