Blame view

slapos/test/test_runner_exporter.py 7.8 KB
Nicolas Wavrant committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
import mock
import os
import shutil
import time
import unittest

from slapos.resilient import runner_exporter
from StringIO import StringIO

tested_instance_cfg = """[buildout]
installed_develop_eggs = 
parts = folders hello-nicolas hello-rafael exclude

[folders]
__buildout_installed__ = 
__buildout_signature__ = wcwidth-0.1.7-py2.7.egg contextlib2-0.5.5-py2.7.egg ...
etc = /some/prefix/slappart18/test/etc
home = /srv/slapgrid/slappart18/test
recipe = slapos.cookbook:mkdirectory
srv = /some/prefix/slappart18/test/srv

[hello-nicolas]
__buildout_installed__ = {cwd}/instance/slappart0/etc/nicolas.txt
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
mode = 0644
name = Nicolas
output = /some/prefix/slappart18/test/etc/nicolas.txt
recipe = slapos.recipe.template

[hello-rafael]
__buildout_installed__ = {cwd}/instance/slappart0/etc//rafael.txt
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
name = Rafael
output = /some/prefix/slappart18/test/etc/rafael.txt
recipe = slapos.recipe.template

[exclude]
__buildout_installed__ = {cwd}/instance/slappart0/srv/exporter.exclude
__buildout_signature__ = MarkupSafe-1.0-py2.7-linux-x86_64.egg Jinja2-2.10-py2.7.egg zc.buildout-2.12.2-py2.7.egg slapos.recipe.template-4.3-py2.7.egg setuptools-40.4.3-py2.7.egg
recipe = slapos.recipe.template:jinja2
rendered = /some/prefix/slappart18/test/srv/exporter.exclude
template = inline:
        srv/backup/**"""


class Config():
  pass


class TestRunnerExporter(unittest.TestCase):
  def setUp(self):
    if not os.path.exists('test_folder'):
      os.mkdir('test_folder')
      os.chdir('test_folder')


  def tearDown(self):
    if os.path.basename(os.getcwd()) == 'test_folder':
      os.chdir('..')
      shutil.rmtree('test_folder')
    elif 'test_folder' in os.listdir('.'):
      shutil.rmtree('test_folder')

  def _createFile(self, path, content=''):
    with open(path, 'w') as f:
      f.write(content)

  def _createExecutableFile(self, path, content=''):
    self._createFile(path, content)
    os.chmod(path, 0700)

  def _setUpFakeInstanceFolder(self):
    self._createFile('proxy.db')
    os.makedirs('project')
    os.makedirs('public')

    """Create data mirroring tested_instance_cfg"""
    os.makedirs('instance/slappart0/etc')
    os.makedirs('instance/slappart0/srv/backup')

    os.makedirs('instance/slappart1/etc')
    os.makedirs('instance/slappart1/srv/backup')

    self._createFile('instance/slappart0/.installed.cfg',
                     tested_instance_cfg.format(cwd=os.getcwd()))

    self._createFile('instance/slappart0/srv/backup/data.dat',
                     'all my fortune lays on this secret !')
    self._createFile('instance/slappart0/srv/exporter.exclude',
                     'srv/backup/**')

    self._createFile('instance/slappart0/etc/config.json')
    self._createFile('instance/slappart0/etc/.parameters.xml')
    self._createFile('instance/slappart0/etc/.project',
                     'workspace/slapos-dev/software/erp5')

    self._createExecutableFile(
      'instance/slappart1/srv/.backup_identity_script',
      "#!/bin/sh\nexec xargs -0 md5sum"
    )


  def test_CwdContextManager(self):
    os.makedirs('a/b')
    with runner_exporter.CwdContextManager('a'):
      self.assertEqual(os.listdir('.'), ['b'])
      os.mkdir('c')
    self.assertEqual(os.listdir('.'), ['a'])
    self.assertEqual(sorted(os.listdir('a')), ['b', 'c'])


  def test_getExcludePathList(self):
    self._setUpFakeInstanceFolder()
    self.assertEqual(
      sorted(runner_exporter.getExcludePathList(os.getcwd())),
      [
        '*.pid',
        '*.sock',
        '*.socket',
        '.installed*.cfg',
        'instance/slappart0/etc/nicolas.txt',
        'instance/slappart0/etc/rafael.txt',
        'instance/slappart0/srv/backup/**',
        'instance/slappart0/srv/exporter.exclude',
      ]
    )


  @mock.patch('subprocess.check_output')
  def test_synchroniseRunnerConfigurationDirectory(self, check_output_mock):
    self._setUpFakeInstanceFolder()
    config = Config()
    config.rsync_binary = 'rsync'
    config.dry = False
    with runner_exporter.CwdContextManager('instance/slappart0/etc'):
      runner_exporter.synchroniseRunnerConfigurationDirectory(
        config, 'backup/runner/etc/'
      )
    self.assertEqual(check_output_mock.call_count, 1)
    check_output_mock.assert_any_call(
Łukasz Nowak committed
141
      ['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', '.parameters.xml', '.project', 'config.json', 'backup/runner/etc/']
Nicolas Wavrant committed
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
    )


  @mock.patch('subprocess.check_output')
  def test_synchroniseRunnerWorkingDirectory(self, check_output_mock):
    self._setUpFakeInstanceFolder()
    config = Config()
    config.rsync_binary = 'rsync'
    config.dry = False
    with runner_exporter.CwdContextManager(os.getcwd()):
      runner_exporter.synchroniseRunnerWorkingDirectory(
        config, 'backup/runner/runner'
      )

    self.assertEqual(check_output_mock.call_count, 1)
    check_output_mock.assert_any_call(
Łukasz Nowak committed
158
      ['rsync', '-rlptgov', '--stats', '--safe-links', '--ignore-missing-args', '--delete', '--delete-excluded', '--exclude=*.pid', '--exclude=*.sock', '--exclude=*.socket', '--exclude=.installed*.cfg', '--exclude=instance/slappart0/etc/nicolas.txt', '--exclude=instance/slappart0/etc/rafael.txt', '--exclude=instance/slappart0/srv/backup/**', '--exclude=instance/slappart0/srv/exporter.exclude', 'instance', 'project', 'proxy.db', 'public', 'backup/runner/runner']
Nicolas Wavrant committed
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
    )

  def test_getSlappartSignatureMethodDict(self):
    self._setUpFakeInstanceFolder()
    slappart_signature_method_dict = runner_exporter.getSlappartSignatureMethodDict()
    self.assertEqual(
      slappart_signature_method_dict,
      {
        './instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
      }
    )
    

  def test_writeSignatureFile(self):
    self._setUpFakeInstanceFolder()

Nicolas Wavrant committed
175 176 177
    os.makedirs('backup/etc')
    os.makedirs('backup/runner/instance/slappart0')
    os.makedirs('backup/runner/instance/slappart1')
Nicolas Wavrant committed
178

Nicolas Wavrant committed
179 180 181
    self._createFile('backup/etc/.project', 'workspace/slapos-dev/software/erp5')
    self._createFile('backup/runner/instance/slappart0/data', 'hello')
    self._createFile('backup/runner/instance/slappart1/data', 'world')
Nicolas Wavrant committed
182 183 184 185 186 187 188 189 190 191 192 193

    slappart_signature_method_dict = {
      './instance/slappart1': './instance/slappart1/srv/.backup_identity_script',
    }

    with runner_exporter.CwdContextManager('backup'):
      runner_exporter.writeSignatureFile(slappart_signature_method_dict, '..', signature_file_path='backup.signature')

      with open('backup.signature', 'r') as f:
        signature_file_content = f.read()

    # Slappart1 is using md5sum as signature, others are using sha256sum (default)
Nicolas Wavrant committed
194 195 196
    self.assertEqual(signature_file_content, """2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824  ./runner/instance/slappart0/data
49b74873d57ff0307b7c9364e2fe2a3876d8722fbe7ce3a6f1438d47647a86f4  ./etc/.project
7d793037a0760186574b0282f2f435e7  ./runner/instance/slappart1/data""")
Nicolas Wavrant committed
197

Łukasz Nowak committed
198
  def test_getBackupFilesModifiedDuringExportList(self):
Nicolas Wavrant committed
199 200
    self._setUpFakeInstanceFolder()
    with runner_exporter.CwdContextManager('instance'):
Łukasz Nowak committed
201 202 203 204
      self.assertEqual(
        runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 5),
        ['./slappart0/srv/backup/data.dat']
      )
Nicolas Wavrant committed
205
      time.sleep(2)
Łukasz Nowak committed
206 207 208 209
      self.assertEqual(
        runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 1),
        []
      )
Nicolas Wavrant committed
210
      self._createFile('slappart1/srv/backup/bakckup.data', 'my backup')
Łukasz Nowak committed
211 212 213 214
      self.assertEqual(
        runner_exporter.getBackupFilesModifiedDuringExportList(time.time() - 1),
        ['./slappart1/srv/backup/bakckup.data']
      )