Update Release Candidate

component/ca-certificates/buildout.cfg

@@ -12,11 +12,11 @@ parts =
 [ca-certificates]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20190110.tar.xz
-md5sum = e91d3d9259127ba2dbb65fda58d73f31
+url = http://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119.tar.xz
+md5sum = c02582bf9ae338e558617291897615eb
 patch-binary = ${patch:location}/bin/patch
 patches =
-  ${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
+  ${:_profile_base_location_}/ca-certificates-any-python.patch#47c2509f1346bd5af8123fb1a2751c2c
   ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
 patch-options = -p0
 configure-command = true

component/ca-certificates/ca-certificates-any-python.patch

@@ -15,7 +15,7 @@
  #
  
  all:
--	python certdata2pem.py
+-	python3 certdata2pem.py
 +	for x in '' 2 3; do type python$$x && break; done >/dev/null \
 +	&& python$$x certdata2pem.py
  

component/defaults.cfg

@@ -34,7 +34,7 @@ depends =
   ${patch:recipe}
 recipe = slapos.recipe.build
 # Latest version provided by SlapOS.
-part = gcc-8.4
+part = gcc-8.5
 # Minimum version for all components that might be required for
 # slapos.rebootstrap (see https://bugs.python.org/issue34112 about Python 3.7+).
 min_version = 5.4

component/gcc/buildout.cfg

@@ -10,7 +10,7 @@ extends =
   ../binutils/buildout.cfg
 
 parts =
-  gcc-8.4
+  gcc-8.5
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
@@ -75,10 +75,10 @@ patches =
   ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
   ${:_profile_base_location_}/glibc2.30-ipc_perm.patch#563b7f5a38c2ea6bf741c328422b4c98
 
-[gcc-8.4]
+[gcc-8.5]
 <= gcc-common
-version = 8.4.0
-md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
+version = 8.5.0
+md5sum = 0c1f625768840187ef3b10adebe8e3b0
 
 [gcc-10.2]
 <= gcc-common

component/gdbm/buildout.cfg

 [buildout]
-extends =
-  ../patch/buildout.cfg
 parts =
   gdbm
 
 [gdbm]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 1.18
-url = ftp://ftp.gnu.org/gnu/gdbm/gdbm-${:version}.tar.gz
-md5sum = e316f8e4a3e7e4f23955be65d54fec48
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/extern_parseopt.patch#30baeaaa0c6a6e6be8cf56a875726619
+version = 1.19
+url = http://ftp.gnu.org/gnu/gdbm/gdbm-${:version}.tar.gz
+md5sum = aeb29c6a90350a4c959cd1df38cd0a7e
 configure-options =
   --disable-static
   --enable-libgdbm-compat
-environment =
-  PATH=${patch:location}/bin:%(PATH)s
-# install as parts/gdbm/include/gdbm/*.h etc. because some softwares
-# (eg. python's dbmmodule.c extension) assume the location like this.
-includedir = @@LOCATION@@/include
-make-targets =
-  install includedir=${:includedir}/gdbm && rm -f ${:includedir}/*.h && ln -sf gdbm/gdbm.h ${:includedir}/gdbm.h

component/gdbm/extern_parseopt.patch

---- a/src/parseopt.c
-+++ b/src/parseopt.c
-@@ -255,8 +255,8 @@
- }
- 
- char *parseopt_program_name;
--char *parseopt_program_doc;
--char *parseopt_program_args;
-+extern char *parseopt_program_doc;
-+extern char *parseopt_program_args;
- const char *program_bug_address = "<" PACKAGE_BUGREPORT ">";
- void (*parseopt_help_hook) (FILE *stream);
- 

component/gettext/buildout.cfg

@@ -11,8 +11,8 @@ extends =
 [gettext]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.8.1.tar.lz
-md5sum = d838d2c4144261d0c5fbab4a0aceb5c1
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.21.tar.lz
+md5sum = 65c9a4d0a2d180e3c2afc2b3440382ef
 
 configure-options =
   --disable-static

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/groonga/groonga-11.0.0.tar.gz
-md5sum = 2ea277c19f21b9d45410547173ff41c2
+url = https://packages.groonga.org/source/groonga/groonga-11.0.2.tar.gz
+md5sum = 753ba6fad77598baf93615c4b9c535b1
 groonga-plugin-dir =  @@LOCATION@@/lib/groonga/plugins/
 # temporary patch to respect more tokens in natural language mode.
 patches =

component/jupyter-py2/buildout.cfg

@@ -16,6 +16,7 @@ parts +=
 max_version = 0
 
 [jupyter]
+extra-eggs =
 python_executable = ${buildout:bin-directory}/${:interpreter}
 
 [download-file-base]
@@ -66,7 +67,6 @@ context =
 Pygments = 2.2.0
 astor = 0.5
 backports-abc = 0.5
-backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
 cycler = 0.10.0
 ipykernel = 4.5.2
@@ -122,11 +122,6 @@ pexpect = 4.2.1
 # ipython==5.3.0
 pickleshare = 0.7.4
 
-# Required by:
-# matplotlib==2.1.2
-# pandas==0.19.2
-python-dateutil = 2.6.0
-
 # Required by:
 # pathlib2==2.2.1
 scandir = 1.5

component/jupyter/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../numpy/openblas.cfg
   ../matplotlib/buildout.cfg
   ../ipython/buildout.cfg
+  ../python-cffi/buildout.cfg
   ../python-pyzmq/buildout.cfg
   ../scipy/buildout.cfg
   ../scikit-learn/buildout.cfg
@@ -14,6 +15,11 @@ parts =
   jupyter
   jupyter-notebook-scripts
 
+[argon2-cffi]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = ${python-cffi:egg}
+
 [jupyter-env]
 <= numpy-env
 
@@ -46,6 +52,9 @@ eggs =
 # additional eggs for jupyterlab
   jupyterlab
   jupyterlab-launcher
+  ${:extra-eggs}
+extra-eggs =
+  ${argon2-cffi:egg}
 
 interpreter = pythonjupyter
 

component/libexpat/buildout.cfg

@@ -7,8 +7,8 @@ parts =
 [libexpat]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/libexpat/libexpat/releases/download/R_2_2_9/expat-2.2.9.tar.lz
-md5sum = c356e4f2092df4f0b0ffef904f001842
+url = https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.lz
+md5sum = 074cab91a2f27dc8cd6c282effaa8ffd
 configure-options =
   --disable-static
   --without-xmlwf

component/libffi/buildout.cfg

@@ -8,13 +8,13 @@ parts =
 [libffi]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 3.2.1
-url = ftp://sourceware.org/pub/libffi/libffi-${:version}.tar.gz
-md5sum = 83b89587607e3eb65c70d361f13bab43
+version = 3.3
+url = http://sourceware.org/pub/libffi/libffi-${:version}.tar.gz
+md5sum = 6313289e32f1d38a9df4770b014a2ca7
 location = @@LOCATION@@
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/includedir.diff#e430307d16a0d215a24ec2acff23d184
+  ${:_profile_base_location_}/libdir.patch#f7274250c8ca43b10e7fc968f730b833
 configure-options =
   --disable-static
   --enable-portable-binary

component/libffi/includedir.diff

-Index: b/libffi.pc.in
-===================================================================
---- a/libffi.pc.in
-+++ b/libffi.pc.in
-@@ -2,10 +2,10 @@ prefix=@prefix@
- exec_prefix=@exec_prefix@
- libdir=@libdir@
- toolexeclibdir=@toolexeclibdir@
--includedir=${libdir}/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
-+includedir=@includedir@
- 
- Name: @PACKAGE_NAME@
- Description: Library supporting Foreign Function Interfaces
- Version: @PACKAGE_VERSION@
--Libs: -L${toolexeclibdir} -lffi
-+Libs: -L${libdir} -lffi
- Cflags: -I${includedir}
-Index: b/include/Makefile.am
-===================================================================
---- a/include/Makefile.am
-+++ b/include/Makefile.am
-@@ -5,5 +5,5 @@ AUTOMAKE_OPTIONS=foreign
- DISTCLEANFILES=ffitarget.h
- EXTRA_DIST=ffi.h.in ffi_common.h
- 
--includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
-+includesdir = $(includedir)
- nodist_includes_HEADERS = ffi.h ffitarget.h
-Index: b/include/Makefile.in
-===================================================================
---- a/include/Makefile.in
-+++ b/include/Makefile.in
-@@ -314,7 +314,7 @@ top_srcdir = @top_srcdir@
- AUTOMAKE_OPTIONS = foreign
- DISTCLEANFILES = ffitarget.h
- EXTRA_DIST = ffi.h.in ffi_common.h
--includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
-+includesdir = $(includedir)
- nodist_includes_HEADERS = ffi.h ffitarget.h
- all: all-am
- 

component/libffi/libdir.patch

+--- a/libffi.pc.in
++++ b/libffi.pc.in
+@@ -7,5 +7,5 @@ prefix=@prefix@
+ Name: @PACKAGE_NAME@
+ Description: Library supporting Foreign Function Interfaces
+ Version: @PACKAGE_VERSION@
+-Libs: -L${toolexeclibdir} -lffi
++Libs: -L${libdir} -lffi
+ Cflags: -I${includedir}

component/libxml2/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 [libxml2]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://xmlsoft.org/sources/libxml2-2.9.9.tar.gz
-md5sum = c04a5a0a042eaa157e8e8c9eabe76bd6
+url = http://xmlsoft.org/sources/libxml2-2.9.10.tar.gz
+md5sum = 10942a1dc23137a8aa07f0639cbfece5
 configure-options =
   --disable-static
   --without-python

component/lunzip/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [lunzip]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://download.savannah.gnu.org/releases-redirect/lzip/lunzip/lunzip-1.7.tar.gz
-md5sum = cb37bd46590a78bd19a77748b3a097ee
+url = http://download.savannah.gnu.org/releases-redirect/lzip/lunzip/lunzip-1.12.tar.gz
+md5sum = 09caf2475c58aa40e94b599a4f7a2d13
 configure-options =
   --disable-static

component/mariadb/buildout.cfg

@@ -30,8 +30,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
-version = 10.4.18
-md5sum = 22e429d532fcfcb0e4beba64e1c296f0
+version = 10.4.19
+md5sum = bf60c7a3feac5854745cd1ad5133f09a
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -96,8 +96,8 @@ post-install =
 # as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/mroonga/mroonga-11.00.tar.gz
-md5sum = 0aefcbb3c0dc6e29c53c6ee6287114b8
+url = https://packages.groonga.org/source/mroonga/mroonga-11.02.tar.gz
+md5sum = 0729c74efc92bfc404b88597488d07e9
 pre-configure = 
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
@@ -131,8 +131,8 @@ environment =
 ### (we just override here for easier revert)
 [mariadb-10.3]
 <= mariadb-10.4
-version = 10.3.27
-md5sum = 6ab2934a671191d8ca8730e9a626c5c9
+version = 10.3.29
+md5sum = a5adad1c4fb1717d7fe6d608fd4d40de
 post-install =
   ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
   for x in ${lz4:location} ${snappy:location} ${zstd:location}

component/python-cachecontrol/buildout.cfg

-[buildout]
-
-[python-cachecontrol]
-recipe = zc.recipe.egg:custom
-egg = cachecontrol
-setup-eggs =
-  pbr
-  lockfile

component/python-cliff/buildout.cfg

-[buildout]
-
-[python-cliff]
-recipe = zc.recipe.egg:custom
-egg = cliff
-setup-eggs =
-  pbr

component/slapos/buildout.cfg

@@ -77,8 +77,6 @@ eggs =
   ${lxml-python:egg}
   ${python-cffi:egg}
   ${python-PyYAML:egg}
-  ${python-cachecontrol:egg}
-  ${python-cliff:egg}
   ${python-cryptography:egg}
   pyOpenSSL
   slapos.libnetworkcache

component/sqlite3/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 [sqlite3]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://sqlite.org/2020/sqlite-autoconf-3320300.tar.gz
-md5sum = 2e3911a3c15e85c2f2d040154bbe5ce3
+url = https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz
+md5sum = d1d1aba394c8e0443077dc9f1a681bb8
 configure-options =
   --disable-static
   --enable-readline

software/backupserver/software.cfg

@@ -32,7 +32,7 @@ recipe = zc.recipe.egg
 interpreter = python-${:_buildout_section_name_}
 eggs =
   PyRSS2Gen
-  ${python-dateutil:egg}
+  python-dateutil
 
 ##########################################################
 # Service startup scripts and configuration files

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = a6a626fd1579fd1d4b80ea67433ca16a
+md5sum = dd6542a91746e6e3720eaa1590792dbe
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 1248911409cbeea980a838b04ee451d2
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 8ce1d5bf09662d941f940be7e6493918
+md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -508,7 +508,7 @@ filename = logging.yaml
 {%- raw %}
 template = inline:
   map /HTTPS/ http://{{ ipv4 }}:{{ https_port }}
-  map / http://{{ ipv4 }}:{{ http_port }}
+  map /HTTP/ http://{{ ipv4 }}:{{ http_port }}
 {%- endraw %}
 extra-context =
   raw ipv4 {{ instance_parameter_dict['ipv4-random'] }}

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -5,7 +5,7 @@
 {%- set frontend_slave_list = [] %}
 {%- set part_list = [] %}
 {%- set cache_port = caddy_configuration.get('cache-port') %}
-{%- set cache_access = "http://%s:%s" % (instance_parameter_dict['ipv4-random'], cache_port) %}
+{%- set cache_access = "http://%s:%s/HTTP" % (instance_parameter_dict['ipv4-random'], cache_port) %}
 {%- set ssl_cache_access = "http://%s:%s/HTTPS" % (instance_parameter_dict['ipv4-random'], cache_port) %}
 {%- set backend_haproxy_http_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['http-port']) %}
 {%- set backend_haproxy_https_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['https-port']) %}

software/caddy-frontend/test/test.py

@@ -1485,6 +1485,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'enable_cache': True,
         'disable-via-header': True,
       },
+      'enable_cache-https-only': {
+        'url': cls.backend_url,
+        'https-only': False,
+        'enable_cache': True,
+      },
       'enable-http2-false': {
         'url': cls.backend_url,
         'enable-http2': False,
@@ -1681,9 +1686,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
       'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
       'domain': 'example.com',
-      'accepted-slave-amount': '50',
+      'accepted-slave-amount': '51',
       'rejected-slave-amount': '0',
-      'slave-amount': '50',
+      'slave-amount': '51',
       'rejected-slave-dict': {
       },
       'warning-slave-dict': {
@@ -3669,6 +3674,61 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       result.headers['Location']
     )
 
+  def test_enable_cache_https_only(self):
+    parameter_dict = self.assertSlaveBase('enable_cache-https-only')
+
+    result = fakeHTTPSResult(
+      parameter_dict['domain'],
+      'test-path/deep/.././deeper', headers={
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
+
+    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+
+    headers = result.headers.copy()
+
+    self.assertKeyWithPop('Server', headers)
+    self.assertKeyWithPop('Date', headers)
+    self.assertKeyWithPop('Age', headers)
+
+    # drop keys appearing randomly in headers
+    headers.pop('Transfer-Encoding', None)
+    headers.pop('Content-Length', None)
+    headers.pop('Connection', None)
+    headers.pop('Keep-Alive', None)
+
+    self.assertEqual(
+      {
+        'Content-type': 'application/json',
+        'Set-Cookie': 'secured=value;secure, nonsecured=value',
+        'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
+                         'stale-if-error=3600'
+      },
+      headers
+    )
+
+    result = fakeHTTPResult(
+      parameter_dict['domain'],
+      'HTTPS/test', headers={
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
+
+    self.assertEqual(httplib.OK, result.status_code)
+    self.assertEqualResultJson(result, 'Path', '/HTTPS/test')
+
+    headers = result.headers.copy()
+
+    result = fakeHTTPSResult(
+      parameter_dict['domain'],
+      'HTTP/test', headers={
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
+
+    self.assertEqual(httplib.OK, result.status_code)
+    self.assertEqualResultJson(result, 'Path', '/HTTP/test')
+
+    headers = result.headers.copy()
+
   def test_enable_cache(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
 

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -55,6 +55,9 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
+T-2/var/log/httpd/_enable_cache-https-only_access_log
+T-2/var/log/httpd/_enable_cache-https-only_backend_log
+T-2/var/log/httpd/_enable_cache-https-only_error_log
 T-2/var/log/httpd/_enable_cache_access_log
 T-2/var/log/httpd/_enable_cache_backend_log
 T-2/var/log/httpd/_enable_cache_custom_domain_access_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -55,6 +55,9 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
 T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
+T-2/var/log/httpd/_enable_cache-https-only_access_log
+T-2/var/log/httpd/_enable_cache-https-only_backend_log
+T-2/var/log/httpd/_enable_cache-https-only_error_log
 T-2/var/log/httpd/_enable_cache_access_log
 T-2/var/log/httpd/_enable_cache_backend_log
 T-2/var/log/httpd/_enable_cache_custom_domain_access_log

software/erp5testnode/software.cfg

@@ -46,7 +46,6 @@ dnspython = 1.15.0
 PyXML = 0.8.5
 WebOb = 1.8.5
 WebTest = 2.0.33
-backports.functools-lru-cache = 1.5
 soupsieve = 1.8
 #gitdb2 = 2.0.4
 #smmap2 = 2.0.4

software/jupyter/software.cfg

@@ -54,13 +54,12 @@ Pygments = 2.7.2
 astor = 0.5
 async-generator = 1.10
 backports-abc = 0.5
-backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
 bleach = 3.2.1
 cycler = 0.10.0
 defusedxml = 0.6.0
 entrypoints = 0.3
-ipykernel = 5.3.4
+ipykernel = 5.3.4:whl
 ipython = 5.3.0
 ipython-genutils = 0.1.0
 ipywidgets = 6.0.0
@@ -130,10 +129,6 @@ pickleshare = 0.7.4
 # notebook==6.1.5
 prometheus-client = 0.9.0
 
-# Required by:
-# statsmodels==0.11.1
-python-dateutil = 2.8.1
-
 # Required by:
 # pathlib2==2.2.1
 scandir = 1.5

software/jupyter/test/test.py

@@ -32,7 +32,7 @@ import os
 import requests
 import sqlite3
 
-
+from slapos.proxy.db_version import DB_VERSION
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -169,27 +169,21 @@ class SelectMixin(object):
     )
     return sqlite3.connect(sqlitedb_file)
 
-  def select(self, fields, table, where={}):
-    connection = self.sqlite3_connect()
-
-    def dict_factory(cursor, row):
-      d = {}
-      for idx, col in enumerate(cursor.description):
-        d[col[0]] = row[idx]
-      return d
-    connection.row_factory = dict_factory
-    cursor = connection.cursor()
-
-    condition = " AND ".join("%s='%s'" % (k, v) for k, v in where.items())
-    cursor.execute(
-      "SELECT %s FROM %s%s"
-      % (
+  def select(self, table, fields=('*',), **where):
+    db = self.sqlite3_connect()
+    try:
+      db.row_factory = lambda cursor, row: {
+        col[0]: row[idx]
+        for idx, col in enumerate(cursor.description)
+      }
+      return db.execute("SELECT %s FROM %s%s%s" % (
         ", ".join(fields),
-        table,
-        " WHERE %s" % condition if where else "",
-      )
-    )
-    return cursor.fetchall()
+        table, DB_VERSION,
+        " WHERE " + " AND ".join("%s='%s'" % x for x in where.items())
+        if where else "",
+      )).fetchall()
+    finally:
+      db.close()
 
 
 class TestJupyterCustomFrontend(SelectMixin, InstanceTestCase):
@@ -226,7 +220,7 @@ class TestJupyterCustomFrontend(SelectMixin, InstanceTestCase):
     except Exception:
       pass
 
-    selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
+    selection = self.select("slave", hosted_by=r._partition_id)
 
     self.assertEqual(len(selection), 1)
 
@@ -268,7 +262,7 @@ class TestJupyterCustomAdditional(SelectMixin, InstanceTestCase):
     except Exception:
       pass
 
-    selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
+    selection = self.select("slave", hosted_by=r._partition_id)
 
     self.assertEqual(len(selection), 1)
 

software/kvm/test/test.py

@@ -46,6 +46,7 @@ import time
 import shutil
 import sys
 
+from slapos.proxy.db_version import DB_VERSION
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.slap.standalone import SlapOSNodeCommandError
@@ -214,22 +215,19 @@ class MonitorAccessMixin(object):
     return sqlite3.connect(sqlitedb_file)
 
   def get_all_instantiated_partition_list(self):
-    connection = self.sqlite3_connect()
-
-    def dict_factory(cursor, row):
-      d = {}
-      for idx, col in enumerate(cursor.description):
-        d[col[0]] = row[idx]
-      return d
-    connection.row_factory = dict_factory
-    cursor = connection.cursor()
-
-    cursor.execute(
-      "SELECT reference, xml, connection_xml, partition_reference, "
-      "software_release, requested_state, software_type "
-      "FROM partition14 "
-      "WHERE slap_state='busy'")
-    return cursor.fetchall()
+    db = self.sqlite3_connect()
+    try:
+      db.row_factory = lambda cursor, row: {
+        col[0]: row[idx]
+        for idx, col in enumerate(cursor.description)
+      }
+      return db.execute(
+        "SELECT reference, xml, connection_xml, partition_reference,"
+              " software_release, requested_state, software_type"
+        " FROM partition%s"
+        " WHERE slap_state='busy'" % DB_VERSION).fetchall()
+    finally:
+      db.close()
 
   def test_access_monitor(self):
     connection_parameter_dict = self.computer_partition\

software/neoppod/software-common.cfg

@@ -45,7 +45,6 @@ environment = neoppod-setup-env
 [neoppod]
 recipe = zc.recipe.egg
 eggs = neoppod[admin, ctl, master]
-  ${python-dateutil:egg}
   ${cython-zstd:egg}
   ${msgpack-python:egg}
   ${:adapter-egg}
@@ -133,7 +132,6 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 setproctitle = 1.1.10
 cython-zstd = 0.2
-python-dateutil = 2.7.3
 
 # Required by:
 # mock = 3.0.5

software/powerdns/software.cfg

@@ -56,6 +56,5 @@ cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 passlib = 1.7.1
 GitPython = 2.1.11
-lockfile = 0.12.2
 feedparser = 5.2.1
 atomize = 0.2.0

software/slapos-sr-testing/software.cfg

@@ -2,7 +2,6 @@
 
 extends =
   ../../component/bcrypt/buildout.cfg
-  ../../component/python-cliff/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/git/buildout.cfg
@@ -202,7 +201,6 @@ setup = ${slapos.core-repository:location}
 [python-interpreter]
 eggs += ${:extra-eggs}
 extra-eggs =
-  ${python-cliff:egg}
   ${lxml-python:egg}
   ${slapos.core-setup:egg}
   ${pillow-python:egg}
@@ -337,7 +335,7 @@ forcediphttpsadapter = 1.0.1
 httplib2 = 0.11.3
 image = 1.5.25
 paramiko = 2.4.2
-plantuml = 0.3.0
+plantuml = 0.3.0:whl
 pysftp = 0.2.9
 requests-toolbelt = 0.8.0
 selenium = 3.141.0
@@ -348,7 +346,7 @@ PyPDF2 = 1.26.0+SlapOSPatched001
 # Django 1.11 is python 2 compatible
 Django = 1.11
 
-mock = 2.0.0
+mock = 2.0.0:whl
 testfixtures = 6.11
 funcsigs = 1.0.2
 mysqlclient = 1.3.12

software/slapos-testing/software.cfg

@@ -234,7 +234,7 @@ dnspython = 1.15.0
 funcsigs = 1.0.2
 httmock = 1.2.6
 manuel = 1.9.0
-mock = 2.0.0
+mock = 2.0.0:whl
 testfixtures = 6.11.0
 pycurl = 7.43.0.2
 pyflakes = 2.0.0

software/slaprunner/software.cfg

@@ -177,4 +177,3 @@ prettytable = 0.7.2
 pycurl = 7.43.0
 collective.recipe.environment = 0.2.0
 smmap = 0.9.0
-lockfile = 0.12.2

software/theia/buildout.hash.cfg

@@ -13,9 +13,13 @@
 # section inheritance (< = ...) are NOT supported (but you should really
 # not need these here).
 
+[instance-theia]
+filename = instance-theia.cfg.jinja.in
+md5sum = 65c66a4cc4eb1d074dcafddf945a34d4
+
 [instance]
 filename = instance.cfg.in
-md5sum = bfd488ba023f505be25d947ec830bab3
+md5sum = 43923e3e1f27c43696ecbca9ee147bdb
 
 [yarn.lock]
 filename = yarn.lock

software/theia/software.cfg

@@ -22,6 +22,7 @@ extends =
 parts =
   theia-wrapper
   slapos-cookbook
+  instance-theia
   instance
 
 # default for slapos-standalone
@@ -323,6 +324,10 @@ template =
   #!/bin/sh
   exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
 
+[instance-theia]
+<= template-base
+output = ${buildout:directory}/instance-theia.cfg.jinja
+
 [instance]
 <= template-base
 output = ${buildout:directory}/instance.cfg

software/theia/test/test.py

@@ -247,6 +247,22 @@ class TestTheiaWithSR(TheiaTestCase):
     self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (self.sr_url, self.sr_type, instance_name), info), info)
 
 
+class TestTheiaFrontend(TheiaTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'additional-frontend-guid': 'SOMETHING'
+    }
+
+  def setUp(self):
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
+
+  def test_http_get(self):
+    for key in ('url', 'additional-url'):
+      resp = requests.get(self.connection_parameters[key], verify=False)
+      self.assertEqual(requests.codes.unauthorized, resp.status_code)
+
+
 class TestTheiaEnv(TheiaTestCase):
   dummy_software_path = os.path.abspath('dummy/software.cfg')
 

software/wendelin/software-kerastensorflow.cfg

@@ -38,5 +38,5 @@ cuda_toolkit_path = /usr/local/cuda
 cudnn_install_path = /usr/local/cuda
 
 [versions]
-mock = 2.0.0
+mock = 2.0.0:whl
 funcsigs = 1.0.2

stack/cloudooo.cfg

@@ -71,7 +71,7 @@ command = true
 
 [versions]
 Paste = 3.4.0
-PasteScript = 3.2.0
+PasteScript = 3.2.0:whl
 WSGIUtils = 0.7.2
 WSGIserver = 1.3
 python-magic = 0.4.18

stack/slapos.cfg

@@ -25,8 +25,6 @@ extends =
   ../component/lxml-python/buildout.cfg
   ../component/python-backports-lzma/buildout.cfg
   ../component/python-cffi/buildout.cfg
-  ../component/python-cliff/buildout.cfg
-  ../component/python-cachecontrol/buildout.cfg
   ../component/python-cryptography/buildout.cfg
   ../component/python-PyYAML/buildout.cfg
   ../component/pycurl/buildout.cfg
@@ -103,11 +101,8 @@ eggs =
   ${lxml-python:egg}
   ${python-cffi:egg}
   ${python-PyYAML:egg}
-  ${python-cachecontrol:egg}
-  ${python-cliff:egg}
   ${python-cryptography:egg}
   ${pyrsistent:egg}
-  ${jsonschema:egg}
 
 # slapos.toolbox containing utilities
 [slapos-toolbox]
@@ -121,36 +116,23 @@ depends = ${slapos-toolbox-dependencies:eggs}
 # slapos.toolbox dependencies.
 recipe = zc.recipe.egg
 eggs =
-  ${python-cliff:egg}
   ${lxml-python:egg}
   ${pycurl:egg}
   ${python-cryptography:egg}
   ${backports.lzma:egg}
-  ${python-dateutil:egg}
-
-[jsonschema]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = setuptools_scm
-
-[python-dateutil]
-recipe = zc.recipe.egg:custom
-egg = ${:_buildout_section_name_}
-setup-eggs = setuptools_scm
 
 # Install a slapos command with networkcache enabled in ${buildout:bin-directory}
 [slapos-command]
 recipe = zc.recipe.egg
 eggs =
-  ${python-cliff:egg}
   ${lxml-python:egg}
   slapos.core
   slapos.libnetworkcache
 
 [versions]
-setuptools = 44.0.0
+setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos010
+zc.buildout = 2.7.1+slapos013
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -165,11 +147,11 @@ Werkzeug = 1.0.1
 ZConfig = 2.9.3
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
-backports.functools-lru-cache = 1.6.1
+backports.functools-lru-cache = 1.6.1:whl
 backports.lzma = 0.0.14
 cffi = 1.14.0
 click = 6.7
-cliff = 2.4.0
+cliff = 2.8.3:whl
 cmd2 = 0.7.0
 collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
@@ -217,7 +199,7 @@ slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.5
 slapos.toolbox = 0.119
-stevedore = 1.21.0
+stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
 unicodecsv = 0.14.1
 wcwidth = 0.2.5
@@ -229,7 +211,7 @@ zipp = 1.2.0
 zodburi = 2.4.0
 zope.event = 3.5.2
 paramiko = 2.1.3
-CacheControl = 0.12.5
+CacheControl = 0.12.6:whl
 msgpack = 0.6.2
 
 # Required by:
@@ -290,21 +272,17 @@ attrs = 18.2.0
 # jsonschema==3.0.2
 pyrsistent = 0.16.1
 
-pytest-runner = 5.2.0
-
-# Required by:
-# jsonschema==3.0.2
-setuptools-scm = 3.5.0
+pytest-runner = 5.2.0:whl
 
 ipaddress = 1.0.23
 
 # Required by:
 # slapos.cookbook==1.0.143
-jsonschema = 3.0.2
+jsonschema = 3.0.2:whl
 
 # Required by:
 # slapos.toolbox==0.94
-lockfile = 0.12.2
+lockfile = 0.12.2:whl
 
 # Required by:
 # slapos.core==1.5.0
@@ -331,7 +309,7 @@ pycurl = 7.43.0
 
 # Required by:
 # slapos.toolbox==0.94
-python-dateutil = 2.7.3
+python-dateutil = 2.7.3:whl
 
 # Required by:
 # slapos.toolbox==0.94