Merge branch 'master' into master+ZODB4-wc2

* master:
  Fix "kvm: Increase default VM parameters to match nowadays requirement"
  kvm: Merge tests
  kvm: Allow to specify network adapter
  software/slapos-sr-testing: install new psycopg2 dependency
  recipe/postgres: support non standard port
  recipe/test_postgres: test with a real postgres server
  software/slapos-testing: install psycopg2 for slapos.cookbook
  software/slapos-testing: make postgresql available for tests
  component/psycopg2: new component
  golang: Switch default to Go1.17
  erp5testnode: update deploy script
  stack/slapos: slapos.cookbook version up (1.0.214)
  Release slapos.cookbook (1.0.214)
  test/recipe: update tests for postgres recipe regarding password update
  recipe/postgres: support changing user password after creation
  component/nxdtest: v↑
  golang: v↑ go1.16 (1.16.8 -> 1.16.10)
  golang: v↑ go1.17 (1.17.1 -> 1.17.3)

component/golang/buildout.cfg

@@ -46,8 +46,8 @@ environment-extra =
 
 [golang1.16]
 <= golang-common
-url = https://golang.org/dl/go1.16.8.src.tar.gz
-md5sum = 92e69a5e1bb6ea5e7498d12d03160032
+url = https://golang.org/dl/go1.16.10.src.tar.gz
+md5sum = 49f0a54f0bdcba297bac194d8dafe431
 
 # go1.16 needs go1.4 to bootstrap
 environment-extra =
@@ -55,8 +55,8 @@ environment-extra =
 
 [golang1.17]
 <= golang-common
-url = https://golang.org/dl/go1.17.1.src.tar.gz
-md5sum = a78205838c2a7054522cb91c12982f26
+url = https://golang.org/dl/go1.17.3.src.tar.gz
+md5sum = 3ea82e5966275f405f0db4f52511bb6e
 
 # go1.17 needs go1.4 to bootstrap
 environment-extra =
@@ -122,7 +122,7 @@ bin	= ${gowork.dir:bin}
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.16:location}
+golang  = ${golang1.17:location}
 
 # no special build flags by default
 buildflags =

component/nxdtest/buildout.cfg

@@ -20,6 +20,6 @@ egg     = nxdtest
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = 0ad45a9c
+revision = 9f413221
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git

component/psycopg2/buildout.cfg

+[buildout]
+extends =
+  ../postgresql/buildout.cfg
+
+[psycopg2-env]
+PATH = ${postgresql:location}/bin:%(PATH)s
+
+[psycopg2]
+recipe = zc.recipe.egg:custom
+egg = psycopg2
+define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+environment = psycopg2-env
+include-dirs =
+  ${postgresql:location}/include
+library-dirs =
+  ${postgresql:location}/lib
+rpath = 
+  ${postgresql:location}/lib

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.213'
+version = '1.0.214'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -39,6 +39,7 @@ extras_require = {
     'test': (
         'jsonschema',
         'mock',
+        'psycopg2',
         'testfixtures',
         'requests',
     ),

slapos/recipe/postgres.py

@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe):
         else:
             paths.extend(self.createConfig())
             paths.extend(self.createRunScript())
+            self.updateSuperuser()
 
         return paths
 
@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe):
         with open(postgres_conf, 'w') as cfg:
             cfg.write(textwrap.dedent("""\
                     listen_addresses = '%s'
+                    port = %s
                     logging_collector = on
                     log_rotation_size = 50MB
                     max_connections = 100
@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe):
                     unix_socket_permissions = 0700
                     """ % (
                         ','.join(set(ipv4).union(ipv6)),
+                        self.options['port'],
                         pgdata,
                         )))
 
@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe):
 
         # encrypt the password to avoid storing in the logs
         enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
+        change_password_query = """ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)
 
-        self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
-
+        pgdata = self.options['pgdata-directory']
+        if os.path.exists(os.path.join(pgdata, 'postmaster.pid')):
+            psql_binary = os.path.join(self.options['bin'], 'psql')
+            # connect to a running postgres deamon
+            p = subprocess.Popen([
+                    psql_binary,
+                    '-h', pgdata,
+                    '-p', self.options['port'],
+                    '-U', user,
+                    '-d', self.options['dbname'],
+                ],
+                stdin=subprocess.PIPE)
+            p.communicate((change_password_query + '\n').encode())
+            if p.returncode:
+                raise UserError("Error updating password")
+        else:
+            self.runPostgresCommand(cmd=change_password_query)
 
     def runPostgresCommand(self, cmd):
         """\

slapos/test/recipe/test_postgres.py

-import unittest
-import tempfile
+import os
 import shutil
-import os.path
+import tempfile
+import textwrap
+import time
+import unittest
+
+try:
+  import subprocess32 as subprocess
+except ImportError:
+  import subprocess
+
+import psycopg2
 import zc.buildout.testing
 
 
 class PostgresTest(unittest.TestCase):
+  ipv4 = os.environ['SLAPOS_TEST_IPV4']
+  ipv6 = os.environ['SLAPOS_TEST_IPV6']
+  port = 5432
+
   def setUp(self):
     self.buildout = buildout = zc.buildout.testing.Buildout()
     self.pgdata_directory = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, self.pgdata_directory)
     self.services_directory = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, self.services_directory)
+
+    self.postgres_bin_directory = os.environ['SLAPOS_TEST_POSTGRESQL_PREFIX'] + '/bin'
     buildout['postgres'] = {
-      'bin': 'software/parts/postgres/bin/',
+      'bin': self.postgres_bin_directory,
       'dbname': 'dbname',
-      'ipv4': '127.0.0.1',
-      'ipv6': '::1',
-      'port': '5443',
-      'pgdata-directory': self.pgdata_directory,
+      'ipv4': self.ipv4,
+      'ipv6': self.ipv6,
+      'port': self.port,
+      'pgdata-directory': os.path.join(self.pgdata_directory, 'pgdata'),
       'services': self.services_directory,
       'superuser': 'superuser',
       'password': 'secret',
@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase):
         'postgres',
         buildout['postgres'])
 
+  def start_postgres_server(self):
+    server_process = subprocess.Popen(
+        [ os.path.join(self.services_directory, 'postgres-start') ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    self.addCleanup(server_process.wait)
+    self.addCleanup(server_process.terminate)
+
+    # wait for server to accept connections
+    for i in range(60):
+      time.sleep(i)
+      try:
+        psycopg2.connect(self.buildout['postgres']['url']).close()
+      except psycopg2.OperationalError as e:
+        pass
+      else:
+        break
+
   def test_options(self):
     self.assertEqual(
-        'postgresql://superuser:secret@[::1]:5443/dbname',
-        self.buildout['postgres']['url'])
+        self.buildout['postgres']['url'],
+        'postgresql://superuser:secret@[{self.ipv6}]:{self.port}/dbname'.format(self=self),
+    )
 
   def test_install(self):
     installed = self.recipe.install()
 
-    self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory))
-    self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory))
+    pgdata_directory = os.path.join(self.pgdata_directory, 'pgdata')
+    self.assertIn('postgresql.conf', os.listdir(pgdata_directory))
+    self.assertIn('pg_hba.conf', os.listdir(pgdata_directory))
     self.assertIn('postgres-start', os.listdir(self.services_directory))
 
     self.assertEqual(
         sorted(installed),
         sorted([
-            os.path.join(self.pgdata_directory, 'postgresql.conf'),
-            os.path.join(self.pgdata_directory, 'pg_hba.conf'),
+            os.path.join(pgdata_directory, 'postgresql.conf'),
+            os.path.join(pgdata_directory, 'pg_hba.conf'),
             os.path.join(self.services_directory, 'postgres-start')]))
+
+    self.start_postgres_server()
+    with psycopg2.connect(self.buildout['postgres']['url']) as cnx:
+      with cnx.cursor() as cursor:
+        cursor.execute("SELECT 1+1")
+        self.assertEqual(cursor.fetchone(), (2,))
+    cnx.close()
+
+  def test_update_password(self):
+    self.recipe.install()
+    self.start_postgres_server()
+    self.recipe.options['password'] = 'new'
+    self.recipe.install()
+
+    dsn = self.buildout['postgres']['url']
+    with psycopg2.connect(psycopg2.extensions.make_dsn(dsn, password='new')) as cnx:
+      with cnx.cursor() as cursor:
+        cursor.execute("SELECT 1+1")
+        self.assertEqual(cursor.fetchone(), (2,))
+    cnx.close()
+
+    # old password can no longer connect
+    with self.assertRaisesRegexp(
+        psycopg2.OperationalError,
+        'password authentication failed'
+    ):
+      psycopg2.connect(dsn)
+
+
+class PostgresTestNonStandardPort(PostgresTest):
+  port = 5433

software/erp5testnode/testsuite/deploy-test/standalone-local-playbook

@@ -237,7 +237,7 @@ function GetOSVersion {
         os_RELEASE=$(lsb_release -r -s)
         os_UPDATE=""
         os_PACKAGE="rpm"
-        if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
+        if [[ "Debian,Ubuntu,LinuxMint,Raspbian" =~ $os_VENDOR ]]; then
             os_PACKAGE="deb"
         elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
             lsb_release -d -s | grep -q openSUSE
@@ -1036,6 +1036,7 @@ function update_package_repo {
         if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
             # if there are transient errors pulling the updates, that's fine.
             # It may be secondary repositories that we don't really care about.
+            apt_get update --allow-releaseinfo-change || /bin/true
             apt_get update || /bin/true
             REPOS_UPDATED=True
         fi
@@ -1927,6 +1928,7 @@ $XTRACE
 # mode: shell-script
 # End:
 
+
 if [[ $EUID -gt 0 ]]; then
     echo "####################################################"
     echo "#                                                  #"
@@ -1969,7 +1971,7 @@ fi
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``export FORCE=yes``
-if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then
+if [[ ! ${DISTRO} =~ (jessie|stretch|buster|bullseye|xenial|bionic|focal|rhel7) ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
     if [[ "$FORCE" != "yes" ]]; then
         die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -1978,10 +1980,10 @@ fi
 
 # Make sure wheezy backports are available.
 if [[ $DISTRO == "wheezy" ]]; then
-    echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
+    echo "deb http://archive.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
 fi
 
-if is_fedora && [[ $DISTRO == "rhel7" ]]; then
+if is_fedora && [[ $DISTRO =~ rhel7|f20|f19 ]]; then
     # RHEL requires EPEL for many Ansible dependencies
 
     # NOTE: We always remove and install latest -- some environments
@@ -2007,6 +2009,8 @@ failovermethod=priority
 enabled=0
 gpgcheck=0
 EOF
+    yum clean all
+    yum makecache
     yum update -y
     yum_install yum-utils
     # Enable a bootstrap repo.  It is removed after finishing
@@ -2051,18 +2055,26 @@ EOF
     is_package_installed libselinux-python || install_package libselinux-python
 fi
 
+is_package_installed unzip || install_package unzip
 is_package_installed openssl || install_package openssl
-# stick to a specific ansible package version
-if is_ubuntu; then
-  # is_ubuntu also handle debian distro
-  is_package_installed ansible || install_package ansible=1.*
-elif is_fedora; then
-  # is_fedora also handle CentOS distro
-  is_package_installed ansible || install_package ansible1.9.noarch
+is_package_installed ansible || install_package ansible
+
+if ! is_ubuntu || [[ $DISTRO = wheezy || $DISTRO == jessie || $DISTRO == stretch || $DISTRO == xenial || $DISTRO == bionic ]]; then
+  is_package_installed python-pip || install_package python-pip
 else
-  is_package_installed ansible || install_package ansible
+  # Ansible is using python3 now
+  is_package_installed python3-pip || install_package python3-pip
+fi
+
+if is_ubuntu && [[ $DISTRO == focal || $DISTRO == buster ]]; then
+  # install python2 in distro where python2 is not installed by default and ansible python interpreter is still python2
+  is_package_installed python || install_package python
+fi
+
+if is_fedora && [[ $DISTRO == "f26" ]]; then
+  is_package_installed python2-rpm || install_package python2-rpm
+  is_package_installed yum || install_package yum
 fi
-is_package_installed python-setuptools || install_package python-setuptools
 
 if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
   is_package_installed python-apt || install_package python-apt
@@ -2082,6 +2094,7 @@ fi
 ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
 mkdir -p $ANSIBLE_PLUGIN_LOCATION
 /bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
+from __future__ import absolute_import
 import os
 import time
 import json
@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule):
       else:
         for filename in os.listdir(self.log_path):
           filepath = os.path.join(self.log_path, filename)
-          if os.path.exists(filepath) and os.path.isfile(filepath):
+          if os.path.isfile(filepath):
             os.unlink(filepath)
 
     def writeLog(self, host, category, content):
-      if not self.fd_list.has_key(category):
+      if category not in self.fd_list:
         self.fd_list[category] = open(
           os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
         )
@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule):
       if host == "localhost":
         host = "127.0.0.1" # keep compatibility
 
-      if type(data) == dict:
+      if type(data) is dict:
         if '_ansible_verbose_override' in data:
           # avoid logging extraneous data
           return
 
-        data = data.copy()
         content = json.dumps(data)
 
         if ignore_errors:
@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule):
 
 EOF
 
-ansible localhost -m easy_install -a name=slapcache --connection=local
-ansible localhost -m easy_install -a name=requests --connection=local
+ansible localhost -m pip -a name=setuptools --connection=local
+ansible localhost -m pip -a name=wheel --connection=local
+ansible localhost -m pip -a name=slapcache --connection=local
+ansible localhost -m pip -a name=requests --connection=local
+
+if [ ! -f /usr/share/ansible_plugins/mitogen.zip ]; then
+    wget -O /usr/share/ansible_plugins/mitogen.zip https://shacache.nxdcdn.com/3a935ff257ddc0ad4e0f23d71681e026f14f309f4bed0a8e2a217da9b294be2c676196703f0dde856ece49d711d0221deae70812f035b24aa5cdd0ca02790e85
+    unzip /usr/share/ansible_plugins/mitogen.zip -d /usr/share/ansible_plugins/mitogen/
+    mv /usr/share/ansible_plugins/mitogen/mitogen-*/* /usr/share/ansible_plugins/mitogen/
+fi
+
 
 # Include Additional Functions
 if [ ! -f /etc/opt/slapcache.cfg ]; then

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33
+md5sum = f902dd10cb052ac262a4a96b9362b3a3
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = fcb35c32ef985e3d69a7914711675dcc
+md5sum = 67901b7a97b324aa36851d8e8b43cad6
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = f840e8b7af83982525f66d7ec12b7085
+md5sum = 875261817970d0f83335824373288b9d
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -339,6 +339,22 @@
               "type": "string",
               "default": "host"
             },
+            "network-adapter": {
+              "title": "Network adapter",
+              "description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
+              "type": "string",
+              "default": "virtio-net-pci",
+              "enum": [
+                "virtio-net-pci",
+                "e1000",
+                "ne2k_isa",
+                "ne2k_pci",
+                "pcnet",
+                "rtl8139",
+                "usb-net",
+                "vmxnet3"
+              ]
+            },
             "nbd-host": {
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
-config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
+config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 2)) + 1)) }}
+config-network-adapter = {{ dumps(kvm_parameter_dict.get('network-adapter', 'virtio-net-pci')) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
 {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}

software/kvm/instance-kvm-input-schema.json

@@ -127,6 +127,22 @@
       "type": "string",
       "default": "host"
     },
+    "network-adapter": {
+      "title": "Network adapter",
+      "description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
+      "type": "string",
+      "default": "virtio-net-pci",
+      "enum": [
+        "virtio-net-pci",
+        "e1000",
+        "ne2k_isa",
+        "ne2k_pci",
+        "pcnet",
+        "rtl8139",
+        "usb-net",
+        "vmxnet3"
+      ]
+    },
     "nbd-host": {
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",

software/kvm/instance-kvm.cfg.jinja2

@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format}
 disk-device-path = ${slap-parameter:disk-device-path}
 disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
 
+network-adapter = ${slap-parameter:network-adapter}
+
 pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
@@ -1075,6 +1077,8 @@ disk-type = virtio
 disk-format = qcow2
 disk-device-path =
 
+network-adapter = virtio-net-pci
+
 cpu-count = 2
 disk-cache =
 disk-aio =

software/kvm/template/template-kvm-run.in

@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
 disk_size = {{ repr(parameter_dict["disk-size"]) }}
 disk_type = {{ repr(parameter_dict["disk-type"]) }}
 
+network_adapter = {{ repr(parameter_dict["network-adapter"]) }}
+
 socket_path = '{{ parameter_dict.get("socket-path") }}'
 nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
               {{ parameter_dict.get("nbd-port") }}),
@@ -255,7 +257,7 @@ if use_nat == 'true':
   if use_tap == 'true' and tap_ipv6_addr != '':
     rules += ',ipv6=off'
   nat_network_parameter = ['-netdev', rules,
-          '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, mac_address)]
+          '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, mac_address)]
 if use_tap == 'true':
   number += 1
   vhost = ''
@@ -264,7 +266,7 @@ if use_tap == 'true':
   tap_network_parameter = ['-netdev',
           'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number,
             tap_interface, vhost),
-          '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
+          '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, tap_mac_address)]
 
 smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
 ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)

software/kvm/test/test.py

@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
   config_file_name = 'boot-image-url-select.json'
 
 
-@skipUnlessKvm
-class TestCpuMemMaxDynamic(InstanceTestCase):
-  __partition_reference__ = 'cmm'
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      'cpu-count': 2,
-      'ram-size': 2048
-    }
-
-  def test(self):
-    with open(os.path.join(
-     self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
-      kvm_raw = fh.read()
-    self.assertIn('smp_count = 2', kvm_raw)
-    self.assertIn('smp_max_count = 3', kvm_raw)
-    self.assertIn('ram_size = 2048', kvm_raw)
-    self.assertIn("ram_max_size = '2560'", kvm_raw)
-
-
 @skipUnlessKvm
 class TestNatRules(InstanceTestCase):
   __partition_reference__ = 'nr'
@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s
     self.assertFalse(
       os.path.exists(
         os.path.join(self.destination_directory, 'destination')))
+
+
+@skipUnlessKvm
+class TestParameterDefault(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'pd'
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  def mangleParameterDict(self, parameter_dict):
+    return parameter_dict
+
+  def _test(self, parameter_dict, expected):
+    self.rerequestInstance(self.mangleParameterDict(parameter_dict))
+    self.slap.waitForInstance(max_retry=10)
+    
+    kvm_raw = glob.glob(os.path.join(
+      self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
+    self.assertEqual(len(kvm_raw), 1)
+    kvm_raw = kvm_raw[0]
+    with open(kvm_raw, 'r') as fh:
+      kvm_raw = fh.read()
+    self.assertIn(expected, kvm_raw)
+
+  def test_disk_type_default(self):
+    self._test({}, "disk_type = 'virtio'")
+
+  def test_disk_type_set(self):
+    self._test({'disk-type': 'ide'}, "disk_type = 'ide'")
+
+  def test_network_adapter_default(self):
+    self._test({}, "network_adapter = 'virtio-net-pci")
+
+  def test_network_adapter_set(self):
+    self._test({'network-adapter': 'e1000'}, "network_adapter = 'e1000'")
+
+  def test_cpu_count_default(self):
+    self._test({}, "init_smp_count = 2")
+
+  def test_cpu_count_default_max(self):
+    self._test({}, "smp_max_count = 3")
+
+  def test_cpu_count_set(self):
+    self._test({'cpu-count': 4}, "init_smp_count = 4")
+
+  def test_cpu_count_set_max(self):
+    self._test({'cpu-count': 4}, "smp_max_count = 5")
+
+  def test_ram_size_default(self):
+    self._test({}, "init_ram_size = 4096")
+
+  def test_ram_size_default_max(self):
+    self._test({}, "ram_max_size = '4608'")
+
+  def test_ram_size_set(self):
+    self._test({'ram-size': 2048}, "init_ram_size = 2048")
+
+  def test_ram_size_set_max(self):
+    self._test({'ram-size': 2048}, "ram_max_size = '2560'")
+
+
+@skipUnlessKvm
+class TestParameterResilient(TestParameterDefault):
+  __partition_reference__ = 'pr'
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-resilient'
+
+
+@skipUnlessKvm
+class TestParameterCluster(TestParameterDefault):
+  __partition_reference__ = 'pc'
+
+  parameter_dict = {
+    "disable-ansible-promise": True
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({
+      "kvm-partition-dict": {
+        "KVM0": cls.parameter_dict
+      }
+    })}
+
+  def mangleParameterDict(self, parameter_dict):
+    local_parameter_dict = self.parameter_dict.copy()
+    local_parameter_dict.update(parameter_dict)
+    return {'_': json.dumps({
+      "kvm-partition-dict": {
+        "KVM0": local_parameter_dict
+      }
+    })}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-cluster'

software/slapos-sr-testing/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/pycurl/buildout.cfg
   ../../component/faketime/buildout.cfg
   ../../component/pillow/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../component/python-mysqlclient/buildout.cfg
   ../../component/python-pynacl/buildout.cfg
@@ -242,6 +243,7 @@ extra-eggs =
   ${python-mysqlclient:egg}
   ${backports.lzma:egg}
   ${bcrypt:egg}
+  ${psycopg2:egg}
   slapos.libnetworkcache
   supervisor
   ${slapos.cookbook-setup:egg}
@@ -393,3 +395,4 @@ mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
 typing = 3.7.4.3
+psycopg2 = 2.8.6

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 84bd6729e9b299c457cea2d1be6d05a4
+md5sum = 597c29546519aabe7259e416d0b92095

software/slapos-testing/instance.cfg

@@ -92,6 +92,7 @@ template = inline:
   export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
   export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory}
   export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory}
+  export SLAPOS_TEST_POSTGRESQL_PREFIX=${postgresql:location}
 
 
 [slapos-test-runner-dot-nxdtest]

software/slapos-testing/software.cfg

@@ -17,6 +17,8 @@ extends =
   ../../component/sed/buildout.cfg
   ../../component/grep/buildout.cfg
   ../../component/userhosts/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/caucase/buildout.cfg
   ../../stack/nxdtest.cfg
@@ -66,6 +68,7 @@ egg = slapos.cookbook[test]
 setup = ${slapos.cookbook-repository:location}
 depends =
   ${slapos.core-setup:egg}
+  ${psycopg2:egg}
   ${slapos-cookbook-dependencies:eggs}
 
 [slapos.core-setup]
@@ -245,3 +248,4 @@ pyflakes = 2.0.0
 zope.testing = 4.6.2
 urllib3 = 1.24.1
 pathlib = 1.0.1
+psycopg2 = 2.8.6

stack/slapos.cfg

@@ -190,7 +190,7 @@ setproctitle = 1.1.10
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.213
+slapos.cookbook = 1.0.214
 slapos.core = 1.6.19
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0

stack/unstable/lapp/buildout.cfg

@@ -36,6 +36,7 @@ extends =
   ../../component/stunnel/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/pycurl/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../slapos.cfg
   ../resilient/buildout.cfg
 
@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2
 mode = 0644
 
 
-#----------------
-#-- Postgres driver for Python recipes.
-
-[psycopg2-env]
-PATH = ${postgresql:location}/bin:%(PATH)s
-
-
-[psycopg2]
-recipe = zc.recipe.egg:custom
-egg = psycopg2
-define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
-environment = psycopg2-env
-include-dirs =
-    ${postgresql:location}/include
-library-dirs =
-    ${postgresql:location}/lib
-rpath = 
-    ${postgresql:location}/lib
-
-
 #----------------
 #--
 #-- Optional part allowing applications using this stack to run a custom