##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import six.moves.http_client as httplib
import json
import os
import glob
import hashlib
import psutil
import requests
import six
import slapos.util
import sqlite3
from six.moves.urllib.parse import parse_qs, urlparse
import unittest
import subprocess
import tempfile
import six.moves.socketserver as SocketServer
from six.moves import SimpleHTTPServer
import multiprocessing
import time
import shutil
import sys

from slapos.proxy.db_version import DB_VERSION
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError
from slapos.testing.utils import findFreeTCPPort

has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')

if has_kvm:
  setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
    os.path.abspath(
      os.path.join(os.path.dirname(__file__), '..',
                   'software%s.cfg' % ("-py3" if six.PY3 else ""))))
else:
  setUpModule, InstanceTestCase = None, unittest.TestCase

  class SanityCheckTestCase(unittest.TestCase):
    def test_kvm_sanity_check(self):
      self.fail('This environment is not usable for kvm testing,'
                ' as it lacks kvm_intel kernel module')

bootstrap_common_param_dict = {
    # the bootstrap script is vm-bootstrap
    "bootstrap-script-url":
    "http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078"
    "af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca455"
    "6a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
    "slave-frontend": {
        "slave-frontend-dict": {}
    },
    "authorized-keys": [
        "ssh-rsa %s key_one" % ("A" * 372),
        "ssh-rsa %s key_two" % ("B" * 372),
        "ssh-rsa %s key_three" % ("C" * 372)
    ],
    "fw-restricted-access": "off",
    "fw-authorized-sources": [],
    "fw-reject-sources": ["10.32.0.0/13"]
}

bootstrap_machine_param_dict = {
    "computer-guid": "local",
    "disable-ansible-promise": True,
    "state": "started",
    "auto-ballooning": True,
    "ram-size": 4096,
    "cpu-count": 2,
    "disk-size": 50,
    # Debian 10 image
    "virtual-hard-drive-url":
    "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183d"
    "f6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f0"
    "30573b904d5ab",
    "virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
    "virtual-hard-drive-gzipped": False,
    "hard-drive-url-check-certificate": False,
    "use-tap": True,
    "use-nat": True,
    "nat-restrict-mode": True,
    "enable-vhost": True,
    "external-disk-number": 1,
    "external-disk-size": 100,
    "external-disk-format": "qcow2",
    "enable-monitor": True,
    "keyboard-layout-language": "fr"
}


class KvmMixin(object):
  def getProcessInfo(self):
    hash_value = generateHashFromFiles([
      os.path.join(self.computer_partition_root_path, hash_file)
      for hash_file in [
        'software_release/buildout.cfg',
      ]
    ])
    # find bin/kvm_raw
    kvm_raw_list = glob.glob(
      os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
    self.assertEqual(1, len(kvm_raw_list))  # allow to work only with one
    hash_file_list = [
      kvm_raw_list[0],
      'software_release/buildout.cfg',
    ]
    kvm_hash_value = generateHashFromFiles([
      os.path.join(self.computer_partition_root_path, hash_file)
      for hash_file in hash_file_list
    ])
    with self.slap.instance_supervisor_rpc as supervisor:
      running_process_info = '\n'.join(sorted([
        '%(group)s:%(name)s %(statename)s' % q for q
        in supervisor.getAllProcessInfo()
        if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
    return running_process_info.replace(
      hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')

  def raising_waitForInstance(self, max_retry):
    with self.assertRaises(SlapOSNodeCommandError):
      self.slap.waitForInstance(max_retry=max_retry)

  def rerequestInstance(self, parameter_dict, state='started'):
    software_url = self.getSoftwareURL()
    software_type = self.getInstanceSoftwareType()
    return self.slap.request(
        software_release=software_url,
        software_type=software_type,
        partition_reference=self.default_partition_reference,
        partition_parameter_kw=parameter_dict,
        state=state)


@skipUnlessKvm
class TestInstance(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'i'

  def test(self):
    connection_parameter_dict = self\
      .computer_partition.getConnectionParameterDict()
    present_key_list = []
    assert_key_list = [
     'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
     'tap-ipv4', 'tap-ipv6']
    for k in assert_key_list:
      if k in connection_parameter_dict:
        present_key_list.append(k)
        connection_parameter_dict.pop(k)
    self.assertEqual(
      connection_parameter_dict,
      {
        'ipv6': self._ipv6_address,
        'maximum-extra-disk-amount': '0',
        'monitor-base-url': 'https://[%s]:8026' % (self._ipv6_address,),
        'nat-rule-port-tcp-22': '%s : 10022' % (self._ipv6_address,),
        'nat-rule-port-tcp-443': '%s : 10443' % (self._ipv6_address,),
        'nat-rule-port-tcp-80': '%s : 10080' % (self._ipv6_address,),
      }
    )
    self.assertEqual(set(present_key_list), set(assert_key_list))
    self.assertEqual(
      """i0:6tunnel-10022-{hash}-on-watch RUNNING
i0:6tunnel-10080-{hash}-on-watch RUNNING
i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
i0:kvm-{kvm-hash-value}-on-watch RUNNING
i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED
i0:websockify-{hash}-on-watch RUNNING
i0:whitelist-domains-download-{hash} RUNNING
i0:whitelist-firewall-{hash} RUNNING""",
      self.getProcessInfo()
    )


class MonitorAccessMixin(object):
  def sqlite3_connect(self):
    sqlitedb_file = os.path.join(
      os.path.abspath(
        os.path.join(
          self.slap.instance_directory, os.pardir
        )
      ), 'var', 'proxy.db'
    )
    return sqlite3.connect(sqlitedb_file)

  def get_all_instantiated_partition_list(self):
    db = self.sqlite3_connect()
    try:
      db.row_factory = lambda cursor, row: {
        col[0]: row[idx]
        for idx, col in enumerate(cursor.description)
      }
      return db.execute(
        "SELECT reference, xml, connection_xml, partition_reference,"
              " software_release, requested_state, software_type"
        " FROM partition%s"
        " WHERE slap_state='busy'" % DB_VERSION).fetchall()
    finally:
      db.close()

  def test_access_monitor(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    monitor_setup_url = connection_parameter_dict['monitor-setup-url']
    monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]

    auth = parse_qs(urlparse(monitor_url_with_auth).path)

    # check that monitor-base-url for all partitions in the tree are accessible
    # with published username and password
    partition_with_monitor_base_url_count = 0
    for partition_information in self.get_all_instantiated_partition_list():
      connection_xml = partition_information.get('connection_xml')
      if not connection_xml:
        continue
      connection_dict = slapos.util.xml2dict(
        connection_xml if six.PY3 else connection_xml.encode('utf-8'))
      monitor_base_url = connection_dict.get('monitor-base-url')
      if not monitor_base_url:
        continue
      result = requests.get(
        monitor_base_url, verify=False, auth=(
          auth['username'][0],
          auth['password'][0])
      )

      self.assertEqual(
        httplib.OK,
        result.status_code
      )
      partition_with_monitor_base_url_count += 1
    self.assertEqual(
      self.expected_partition_with_monitor_base_url_count,
      partition_with_monitor_base_url_count
    )


@skipUnlessKvm
class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'ad'
  expected_partition_with_monitor_base_url_count = 1

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    self.assertNotIn('url-additional', connection_parameter_dict)


@skipUnlessKvm
class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'ada'
  expected_partition_with_monitor_base_url_count = 1

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'frontend-additional-instance-guid': 'SOMETHING'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'adb'
  expected_partition_with_monitor_base_url_count = 1
  # as few gigabytes are being downloaded, wait a bit longer
  instance_max_retry = 100

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps(dict(
      bootstrap_common_param_dict, **bootstrap_machine_param_dict))}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akc'
  expected_partition_with_monitor_base_url_count = 2

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True
        }
      }
    })}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    self.assertNotIn('KVM0-url-additional', connection_parameter_dict)


@skipUnlessKvm
class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akca'
  expected_partition_with_monitor_base_url_count = 2

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "frontend": {
        'frontend-additional-instance-guid': 'SOMETHING',
      },
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
        }
      }
    })}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['KVM0-url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akcb'
  expected_partition_with_monitor_base_url_count = 3
  # as few gigabytes are being downloaded, wait a bit longer
  instance_max_retry = 100

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps(dict(bootstrap_common_param_dict, **{
      "kvm-partition-dict": {
          "test-machine1": bootstrap_machine_param_dict,
          "test-machine2": dict(bootstrap_machine_param_dict, **{
              # Debian 9 image
              "virtual-hard-drive-url":
              "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
              "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
              "a508930046a78cffea6e0e29d03345333cc",
              "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
              "virtual-hard-drive-gzipped": True,
          })
      }
    }))}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(
      connection_parameter_dict['test-machine1-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    result = requests.get(
      connection_parameter_dict['test-machine2-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestInstanceResilient(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'ir'
  instance_max_retry = 20

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'

  def test(self):
    connection_parameter_dict = self\
      .computer_partition.getConnectionParameterDict()
    present_key_list = []
    assert_key_list = [
     'monitor-password', 'takeover-kvm-1-password', 'backend-url', 'url',
     'monitor-setup-url', 'ipv6-network-info']
    for k in assert_key_list:
      if k in connection_parameter_dict:
        present_key_list.append(k)
        connection_parameter_dict.pop(k)
    self.assertEqual(
      connection_parameter_dict,
      {
        'feed-url-kvm-1-pull': 'http://[%s]:8088/get/local-ir0-kvm-1-pull' % (
          self._ipv6_address,),
        'feed-url-kvm-1-push': 'http://[%s]:8088/get/local-ir0-kvm-1-push' % (
          self._ipv6_address,),
        'ipv6': self._ipv6_address,
        'monitor-base-url': 'https://[%s]:8160' % (self._ipv6_address,),
        'monitor-user': 'admin',
        'takeover-kvm-1-url': 'http://[%s]:9263/' % (self._ipv6_address,),
      }
    )
    self.assertEqual(set(present_key_list), set(assert_key_list))

    self.assertEqual(
      """ir0:bootstrap-monitor EXITED
ir0:certificate_authority-{hash}-on-watch RUNNING
ir0:crond-{hash}-on-watch RUNNING
ir0:monitor-httpd-{hash}-on-watch RUNNING
ir0:monitor-httpd-graceful EXITED
ir1:bootstrap-monitor EXITED
ir1:certificate_authority-{hash}-on-watch RUNNING
ir1:crond-{hash}-on-watch RUNNING
ir1:equeue-on-watch RUNNING
ir1:monitor-httpd-{hash}-on-watch RUNNING
ir1:monitor-httpd-graceful EXITED
ir1:notifier-on-watch RUNNING
ir1:pbs_sshkeys_authority-on-watch RUNNING
ir2:6tunnel-10022-{hash}-on-watch RUNNING
ir2:6tunnel-10080-{hash}-on-watch RUNNING
ir2:6tunnel-10443-{hash}-on-watch RUNNING
ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING
ir2:kvm-{kvm-hash-value}-on-watch RUNNING
ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED
ir2:notifier-on-watch RUNNING
ir2:resilient_sshkeys_authority-on-watch RUNNING
ir2:sshd-graceful EXITED
ir2:sshd-on-watch RUNNING
ir2:websockify-{hash}-on-watch RUNNING
ir2:whitelist-domains-download-{hash} RUNNING
ir2:whitelist-firewall-{hash} RUNNING
ir3:bootstrap-monitor EXITED
ir3:certificate_authority-{hash}-on-watch RUNNING
ir3:crond-{hash}-on-watch RUNNING
ir3:equeue-on-watch RUNNING
ir3:monitor-httpd-{hash}-on-watch RUNNING
ir3:monitor-httpd-graceful EXITED
ir3:notifier-on-watch RUNNING
ir3:resilient-web-takeover-httpd-on-watch RUNNING
ir3:resilient_sshkeys_authority-on-watch RUNNING
ir3:sshd-graceful EXITED
ir3:sshd-on-watch RUNNING""",
      self.getProcessInfo()
    )


@skipUnlessKvm
class TestAccessResilientAdditional(InstanceTestCase):
  __partition_reference__ = 'ara'
  expected_partition_with_monitor_base_url_count = 1

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'frontend-additional-instance-guid': 'SOMETHING'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


class TestInstanceNbdServer(InstanceTestCase):
  __partition_reference__ = 'ins'
  instance_max_retry = 5

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'nbd'

  @classmethod
  def getInstanceParameterDict(cls):
    # port 8080 is used by testnode, use another one
    return {
      'otu-port': '8090'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(
      connection_parameter_dict['upload_url'].strip(), verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>Upload new File</title>', result.text)
    self.assertIn("WARNING", connection_parameter_dict['status_message'])


class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  def log_message(self, *args):
    if os.environ.get('SLAPOS_TEST_DEBUG'):
      return SimpleHTTPServer.SimpleHTTPRequestHandler.log_message(self, *args)
    else:
      return


class FakeImageServerMixin(KvmMixin):
  def startImageHttpServer(self):
    self.image_source_directory = tempfile.mkdtemp()
    server = SocketServer.TCPServer(
      (self._ipv4_address, findFreeTCPPort(self._ipv4_address)),
      FakeImageHandler)

    fake_image_content = b'fake_image_content'
    self.fake_image_md5sum = hashlib.md5(fake_image_content).hexdigest()
    with open(os.path.join(
      self.image_source_directory, self.fake_image_md5sum), 'wb') as fh:
      fh.write(fake_image_content)

    fake_image2_content = b'fake_image2_content'
    self.fake_image2_md5sum = hashlib.md5(fake_image2_content).hexdigest()
    with open(os.path.join(
      self.image_source_directory, self.fake_image2_md5sum), 'wb') as fh:
      fh.write(fake_image2_content)

    self.fake_image_wrong_md5sum = self.fake_image2_md5sum

    url = 'http://%s:%s' % server.server_address
    self.fake_image = '/'.join([url, self.fake_image_md5sum])
    self.fake_image2 = '/'.join([url, self.fake_image2_md5sum])

    old_dir = os.path.realpath(os.curdir)
    os.chdir(self.image_source_directory)
    try:
      self.server_process = multiprocessing.Process(
        target=server.serve_forever, name='FakeImageHttpServer')
      self.server_process.start()
    finally:
      os.chdir(old_dir)

  def stopImageHttpServer(self):
    self.logger.debug('Stopping process %s' % (self.server_process,))
    self.server_process.join(10)
    self.server_process.terminate()
    time.sleep(0.1)
    if self.server_process.is_alive():
      self.logger.warning(
        'Process %s still alive' % (self.server_process, ))

    shutil.rmtree(self.image_source_directory)


@skipUnlessKvm
class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'biul'
  kvm_instance_partition_reference = 'biul0'

  # variations
  key = 'boot-image-url-list'
  test_input = "%s#%s\n%s#%s"
  image_directory = 'boot-image-url-list-repository'
  config_state_promise = 'boot-image-url-list-config-state-promise.py'
  download_md5sum_promise = 'boot-image-url-list-download-md5sum-promise.py'
  download_state_promise = 'boot-image-url-list-download-state-promise.py'

  bad_value = "jsutbad"
  incorrect_md5sum_value_image = "%s#"
  incorrect_md5sum_value = "url#asdasd"
  single_image_value = "%s#%s"
  unreachable_host_value = "evennotahost#%s"
  too_many_image_value = """
      image1#11111111111111111111111111111111
      image2#22222222222222222222222222222222
      image3#33333333333333333333333333333333
      image4#44444444444444444444444444444444
      image5#55555555555555555555555555555555
      image6#66666666666666666666666666666666
      """

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'default'

  @classmethod
  def getInstanceParameterDict(cls):
    # start with empty, but working configuration
    return {}

  def setUp(self):
    super(InstanceTestCase, self).setUp()
    self.startImageHttpServer()

  def tearDown(self):
    # clean up the instance for other tests
    # 1st remove all images...
    self.rerequestInstance({self.key: ''})
    self.slap.waitForInstance(max_retry=10)
    # 2nd ...move instance to "default" state
    self.rerequestInstance({})
    self.slap.waitForInstance(max_retry=10)
    self.stopImageHttpServer()
    super(InstanceTestCase, self).tearDown()

  def test(self):
    partition_parameter_kw = {
      self.key: self.test_input % (
        self.fake_image, self.fake_image_md5sum, self.fake_image2,
        self.fake_image2_md5sum)
    }
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=10)
    # check that image is correctly downloaded and linked
    kvm_instance_partition = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)
    image_repository = os.path.join(
      kvm_instance_partition, 'srv', self.image_directory)
    image = os.path.join(image_repository, self.fake_image_md5sum)
    image_link = os.path.join(image_repository, 'image_001')
    self.assertTrue(os.path.exists(image))
    with open(image, 'rb') as fh:
      image_md5sum = hashlib.md5(fh.read()).hexdigest()
    self.assertEqual(image_md5sum, self.fake_image_md5sum)
    self.assertTrue(os.path.islink(image_link))
    self.assertEqual(os.readlink(image_link), image)

    image2 = os.path.join(image_repository, self.fake_image2_md5sum)
    image2_link = os.path.join(image_repository, 'image_002')
    self.assertTrue(os.path.exists(image2))
    with open(image2, 'rb') as fh:
      image2_md5sum = hashlib.md5(fh.read()).hexdigest()
    self.assertEqual(image2_md5sum, self.fake_image2_md5sum)
    self.assertTrue(os.path.islink(image2_link))
    self.assertEqual(os.readlink(image2_link), image2)

    def getRunningImageList():
      running_image_list = []
      with self.slap.instance_supervisor_rpc as instance_supervisor:
        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                   if 'kvm-' in q['name']][0]['pid']
        kvm_process = psutil.Process(kvm_pid)
        software_root = '/'.join([
          self.slap.software_directory,
          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
        for entry in kvm_process.cmdline():
          if entry.startswith('file') and 'media=cdrom' in entry:
            # do cleanups
            entry = entry.replace(software_root, '')
            entry = entry.replace(kvm_instance_partition, '')
            running_image_list.append(entry)
      return running_image_list

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    self.assertEqual(
      [
        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
        'media=cdrom'
      ],
      getRunningImageList()
    )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    partition_parameter_kw[self.key] = ''
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=2)
    self.assertEqual(
      os.listdir(image_repository),
      []
    )

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    # again only default image is available in the running process
    self.assertEqual(
      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
       'media=cdrom'],
      getRunningImageList()
    )

  def assertPromiseFails(self, promise):
    partition_directory = os.path.join(
      self.slap.instance_directory,
      self.kvm_instance_partition_reference)
    monitor_run_promise = os.path.join(
      partition_directory, 'software_release', 'bin',
      'monitor.runpromise'
    )
    monitor_configuration = os.path.join(
      partition_directory, 'etc', 'monitor.conf')

    self.assertNotEqual(
      0,
      subprocess.call([
        monitor_run_promise, '-c', monitor_configuration, '-a', '-f',
        '--run-only', promise])
    )

  def test_bad_parameter(self):
    self.rerequestInstance({
      self.key: self.bad_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_incorrect_md5sum(self):
    self.rerequestInstance({
      self.key: self.incorrect_md5sum_value_image % (self.fake_image,)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)
    self.rerequestInstance({
      self.key: self.incorrect_md5sum_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_not_matching_md5sum(self):
    self.rerequestInstance({
      self.key: self.single_image_value % (
        self.fake_image, self.fake_image_wrong_md5sum)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.download_md5sum_promise)
    self.assertPromiseFails(self.download_state_promise)

  def test_unreachable_host(self):
    self.rerequestInstance({
      self.key: self.unreachable_host_value % (
        self.fake_image_md5sum,)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.download_state_promise)

  def test_too_many_images(self):
    self.rerequestInstance({
      self.key: self.too_many_image_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)


@skipUnlessKvm
class TestBootImageUrlListResilient(TestBootImageUrlList):
  kvm_instance_partition_reference = 'biul2'
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList):
  __partition_reference__ = 'bius'
  kvm_instance_partition_reference = 'bius0'

  # variations
  key = 'boot-image-url-select'
  test_input = '["%s#%s", "%s#%s"]'
  image_directory = 'boot-image-url-select-repository'
  config_state_promise = 'boot-image-url-select-config-state-promise.py'
  download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
  download_state_promise = 'boot-image-url-select-download-state-promise.py'

  bad_value = '["jsutbad"]'
  incorrect_md5sum_value_image = '["%s#"]'
  incorrect_md5sum_value = '["url#asdasd"]'
  single_image_value = '["%s#%s"]'
  unreachable_host_value = '["evennotahost#%s"]'
  too_many_image_value = """[
      "image1#11111111111111111111111111111111",
      "image2#22222222222222222222222222222222",
      "image3#33333333333333333333333333333333",
      "image4#44444444444444444444444444444444",
      "image5#55555555555555555555555555555555",
      "image6#66666666666666666666666666666666"
      ]"""

  def test_not_json(self):
    self.rerequestInstance({
      self.key: 'notjson#notjson'
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_together(self):
    partition_parameter_kw = {
      'boot-image-url-list': "%s#%s" % (
        self.fake_image, self.fake_image_md5sum),
      'boot-image-url-select': '["%s#%s"]' % (
        self.fake_image, self.fake_image_md5sum)
    }
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=10)
    # check that image is correctly downloaded and linked
    for image_directory in [
      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
      image_repository = os.path.join(
        self.slap.instance_directory, self.kvm_instance_partition_reference,
        'srv', image_directory)
      image = os.path.join(image_repository, self.fake_image_md5sum)
      image_link = os.path.join(image_repository, 'image_001')
      self.assertTrue(os.path.exists(image))
      with open(image, 'rb') as fh:
        image_md5sum = hashlib.md5(fh.read()).hexdigest()
      self.assertEqual(image_md5sum, self.fake_image_md5sum)
      self.assertTrue(os.path.islink(image_link))
      self.assertEqual(os.readlink(image_link), image)

    kvm_instance_partition = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)

    def getRunningImageList():
      running_image_list = []
      with self.slap.instance_supervisor_rpc as instance_supervisor:
        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                   if 'kvm-' in q['name']][0]['pid']
        kvm_process = psutil.Process(kvm_pid)
        software_root = '/'.join([
          self.slap.software_directory,
          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
        for entry in kvm_process.cmdline():
          if entry.startswith('file') and 'media=cdrom' in entry:
            # do cleanups
            entry = entry.replace(software_root, '')
            entry = entry.replace(kvm_instance_partition, '')
            running_image_list.append(entry)
      return running_image_list

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    self.assertEqual(
      [
        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
        'media=cdrom'
      ],
      getRunningImageList()
    )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    self.rerequestInstance(
      {'boot-image-url-list': '', 'boot-image-url-select': ''})
    self.slap.waitForInstance(max_retry=2)
    for image_directory in [
      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
      image_repository = os.path.join(
        kvm_instance_partition, 'srv', image_directory)
      self.assertEqual(
        os.listdir(image_repository),
        []
      )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    partition_parameter_kw[self.key] = ''
    partition_parameter_kw['boot-image-url-list'] = ''
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=2)
    self.assertEqual(
      os.listdir(image_repository),
      []
    )

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    # again only default image is available in the running process
    self.assertEqual(
      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
       'media=cdrom'],
      getRunningImageList()
    )


@skipUnlessKvm
class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
  kvm_instance_partition_reference = 'bius2'
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'biulkc'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  input_value = "%s#%s"
  key = 'boot-image-url-list'
  config_file_name = 'boot-image-url-list.conf'

  def setUp(self):
    super(InstanceTestCase, self).setUp()
    self.startImageHttpServer()

  def tearDown(self):
    self.stopImageHttpServer()
    super(InstanceTestCase, self).tearDown()

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
        },
        "KVM1": {
            "disable-ansible-promise": True,
        }
      }
    })}

  def test(self):
    # Note: As there is no way to introspect nicely where partition landed
    #       we assume ordering of the cluster requests
    self.rerequestInstance({'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
            self.key: self.input_value % (
              self.fake_image, self.fake_image_md5sum)
        },
        "KVM1": {
            "disable-ansible-promise": True,
            self.key: self.input_value % (
              self.fake_image2, self.fake_image2_md5sum)
        }
      }
    })})
    self.slap.waitForInstance(max_retry=10)
    KVM0_config = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
      self.config_file_name)
    KVM1_config = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
      self.config_file_name)
    with open(KVM0_config, 'r') as fh:
      self.assertEqual(
        self.input_value % (self.fake_image, self.fake_image_md5sum),
        fh.read().strip()
      )
    with open(KVM1_config, 'r') as fh:
      self.assertEqual(
        self.input_value % (self.fake_image2, self.fake_image2_md5sum),
        fh.read().strip()
      )


@skipUnlessKvm
class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
  __partition_reference__ = 'biuskc'

  input_value = "[\"%s#%s\"]"
  key = 'boot-image-url-select'
  config_file_name = 'boot-image-url-select.json'


@skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase):
  __partition_reference__ = 'cmm'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'cpu-count': 2,
      'ram-size': 2048
    }

  def test(self):
    with open(os.path.join(
     self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
      kvm_raw = fh.read()
    self.assertIn('smp_count = 2', kvm_raw)
    self.assertIn('smp_max_count = 3', kvm_raw)
    self.assertIn('ram_size = 2048', kvm_raw)
    self.assertIn("ram_max_size = '2560'", kvm_raw)


@skipUnlessKvm
class TestNatRules(InstanceTestCase):
  __partition_reference__ = 'nr'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'nat-rules': '100 200',
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    self.assertIn('nat-rule-port-tcp-100', connection_parameter_dict)
    self.assertIn('nat-rule-port-tcp-200', connection_parameter_dict)

    self.assertEqual(
      '%s : 10100' % (self._ipv6_address,),
      connection_parameter_dict['nat-rule-port-tcp-100']
    )
    self.assertEqual(
      '%s : 10200' % (self._ipv6_address,),
      connection_parameter_dict['nat-rule-port-tcp-200']
    )


@skipUnlessKvm
class TestNatRulesKvmCluster(InstanceTestCase):
  __partition_reference__ = 'nrkc'

  nat_rules = ["100", "200", "300"]
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "nat-rules": cls.nat_rules,
            "disable-ansible-promise": True,
        }
      }
    })}

  def getRunningHostFwd(self):
    with self.slap.instance_supervisor_rpc as instance_supervisor:
      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                 if 'kvm-' in q['name']][0]['pid']
      kvm_process = psutil.Process(kvm_pid)
      for entry in kvm_process.cmdline():
        if 'hostfwd' in entry:
          return entry

  def test(self):
    host_fwd_entry = self.getRunningHostFwd()
    self.assertIn(
      'hostfwd=tcp:%s:10100-:100' % (self._ipv4_address,),
      host_fwd_entry)
    self.assertIn(
      'hostfwd=tcp:%s:10200-:200' % (self._ipv4_address,),
      host_fwd_entry)
    self.assertIn(
      'hostfwd=tcp:%s:10300-:300' % (self._ipv4_address,),
      host_fwd_entry)


@skipUnlessKvm
class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
  __partition_reference__ = 'nrkcc'
  nat_rules = ["100", "200 300"]


@skipUnlessKvm
class TestWhitelistFirewall(InstanceTestCase):
  __partition_reference__ = 'wf'
  kvm_instance_partition_reference = 'wf0'

  def test(self):
    slapos_whitelist_firewall = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference,
      '.slapos-whitelist-firewall')
    self.assertTrue(os.path.exists(slapos_whitelist_firewall))
    with open(slapos_whitelist_firewall, 'rb') as fh:
      content = fh.read()
    try:
      self.content_json = json.loads(content)
    except ValueError:
      self.fail('Failed to parse json of %r' % (content,))
    self.assertTrue(isinstance(self.content_json, list))
    # check /etc/resolv.conf
    with open('/etc/resolv.conf', 'r') as f:
      resolv_conf_ip_list = []
      for line in f.readlines():
        if line.startswith('nameserver'):
          resolv_conf_ip_list.append(line.split()[1])
    resolv_conf_ip_list = list(set(resolv_conf_ip_list))
    self.assertFalse(len(resolv_conf_ip_list) == 0)
    self.assertTrue(all([q in self.content_json for q in resolv_conf_ip_list]))
    # there is something more
    self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))


@skipUnlessKvm
class TestWhitelistFirewallRequest(TestWhitelistFirewall):
  whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'whitelist-domains': cls.whitelist_domains,
    }

  def test(self):
    super(TestWhitelistFirewallRequest, self).test()
    self.assertIn('2.2.2.2', self.content_json)
    self.assertIn('3.3.3.3', self.content_json)
    self.assertIn('4.4.4.4', self.content_json)


@skipUnlessKvm
class TestWhitelistFirewallResilient(TestWhitelistFirewall):
  kvm_instance_partition_reference = 'wf2'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
  kvm_instance_partition_reference = 'wf2'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestWhitelistFirewallCluster(TestWhitelistFirewall):
  kvm_instance_partition_reference = 'wf1'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True
        }
      }
    })}


@skipUnlessKvm
class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
  kvm_instance_partition_reference = 'wf1'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "whitelist-domains": cls.whitelist_domains,
            "disable-ansible-promise": True
        }
      }
    })}


@skipUnlessKvm
class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'ddpwdo'
  kvm_instance_partition_reference = 'ddpwdo0'

  def test(self):
    self.rerequestInstance({
      'disk-device-path': '/dev/virt0 /dev/virt1',
      'wipe-disk-ondestroy': True
    })
    self.raising_waitForInstance(3)
    instance_path = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)

    slapos_wipe_device_disk = os.path.join(
      instance_path, 'etc', 'prerm', 'slapos_wipe_device_disk')

    # check prerm script, it's trusted that prerm manager really works
    self.assertTrue(os.path.exists(slapos_wipe_device_disk))
    with open(slapos_wipe_device_disk) as fh:
      self.assertEqual(
        fh.read().strip(),
        r"""dd if=/dev/zero of=/dev/virt0 bs=4096 count=500k
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
      )
    self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))


@skipUnlessKvm
class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'idc'
  maxDiff = None

  def setUp(self):
    super(TestImageDownloadController, self).setUp()
    self.working_directory = tempfile.mkdtemp()
    self.destination_directory = os.path.join(
      self.working_directory, 'destination')
    os.mkdir(self.destination_directory)
    self.config_json = os.path.join(
      self.working_directory, 'config.json')
    self.md5sum_fail_file = os.path.join(
      self.working_directory, 'md5sum_fail_file')
    self.error_state_file = os.path.join(
      self.working_directory, 'error_state_file')
    self.processed_md5sum = os.path.join(
      self.working_directory, 'processed_md5sum')
    self.startImageHttpServer()
    self.image_download_controller = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '0',
      'software_release', 'parts', 'image-download-controller',
      'image-download-controller')

  def tearDown(self):
    self.stopImageHttpServer()
    shutil.rmtree(self.working_directory)
    super(InstanceTestCase, self).tearDown()

  def callImageDownloadController(self, *args):
    call_list = [sys.executable, self.image_download_controller] + list(args)
    try:
      return (0, subprocess.check_output(
        call_list, stderr=subprocess.STDOUT).decode('utf-8'))
    except subprocess.CalledProcessError as e:
      return (e.returncode, e.output.decode('utf-8'))

  def runImageDownloadControlerWithDict(self, json_dict):
    with open(self.config_json, 'w') as fh:
      json.dump(json_dict, fh, indent=2)
    return self.callImageDownloadController(
      self.config_json,
      'curl',  # comes from test environemnt, considered to be recent enough
      self.md5sum_fail_file,
      self.error_state_file,
      self.processed_md5sum
    )

  def assertFileContent(self, path, content):
    self.assertTrue(os.path.exists, path)
    with open(path, 'r') as fh:
      self.assertEqual(
        fh.read(),
        content)

  def test(self):
    json_dict = {
      'error-amount': 0,
      'config-md5sum': 'config-md5sum',
      'destination-directory': self.destination_directory,
      'image-list': [
        {
          'destination-tmp': 'tmp',
          'url': self.fake_image,
          'destination': 'destination',
          'link': 'image_001',
          'gzipped': False,
          'md5sum': self.fake_image_md5sum,
        }
      ]
    }
    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
INF: %(fake_image)s : Stored with checksum %(checksum)s
INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
""".strip() % {
        'fake_image': self.fake_image,
        'checksum': self.fake_image_md5sum,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(self.destination_directory, 'destination'),
      })
    )
    self.assertFileContent(self.md5sum_fail_file, '')
    self.assertFileContent(self.error_state_file, '')
    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
    self.assertFalse(
      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
    self.assertFileContent(
      os.path.join(self.destination_directory, 'destination'),
      'fake_image_content'
    )

    # Nothing happens if all is downloaded
    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : already downloaded
""".strip() % {
        'fake_image': self.fake_image,
        'checksum': self.fake_image_md5sum,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(self.destination_directory, 'destination'),
      })
    )

  def test_fail(self):
    json_dict = {
      'error-amount': 0,
      'config-md5sum': 'config-md5sum',
      'destination-directory': self.destination_directory,
      'image-list': [
        {
          'destination-tmp': 'tmp',
          'url': self.fake_image,
          'destination': 'destination',
          'link': 'image_001',
          'gzipped': False,
          'md5sum': self.fake_image_wrong_md5sum,
        }
      ]
    }
    for try_num in range(1, 5):
      code, result = self.runImageDownloadControlerWithDict(
        json_dict
      )
      self.assertEqual(
        (code, result.strip()),
        (1, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
""".  strip() % {
          'fake_image': self.fake_image,
          'error_state_file': self.error_state_file,
          'symlink': os.path.join(self.destination_directory, 'image_001'),
          'destination': os.path.join(
            self.destination_directory, 'destination'),
        })
      )
      fake_image_url = '#'.join([
        self.fake_image, self.fake_image_wrong_md5sum])
      self.assertFileContent(
        self.md5sum_fail_file, """{
  "%s": %s
}""" % (fake_image_url, try_num))
      self.assertFileContent(
        self.error_state_file, """
        ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
        """but got instead %(real_checksum)s""".strip() % {
          'fake_image': self.fake_image,
          'wrong_checksum': self.fake_image_wrong_md5sum,
          'real_checksum': self.fake_image_md5sum,
        })
      self.assertFileContent(self.processed_md5sum, 'config-md5sum')
      self.assertFalse(
        os.path.exists(os.path.join(self.destination_directory, 'tmp')))
      self.assertFalse(
        os.path.exists(
          os.path.join(self.destination_directory, 'destination')))

    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (1, """
INF: Storing errors in %(error_state_file)s
""".  strip() % {
        'fake_image': self.fake_image,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(
          self.destination_directory, 'destination'),
      })
    )
    fake_image_url = '#'.join([
      self.fake_image, self.fake_image_wrong_md5sum])
    self.assertFileContent(
      self.md5sum_fail_file, """{
  "%s": %s
}""" % (fake_image_url, 4))
    self.assertFileContent(
      self.error_state_file, """
      ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
      """retry""".strip() % {
        'fake_image': self.fake_image,
      })
    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
    self.assertFalse(
      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
    self.assertFalse(
      os.path.exists(
        os.path.join(self.destination_directory, 'destination')))
