slapgrid.py 96.9 KB
Newer Older
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

28
from __future__ import absolute_import
Łukasz Nowak's avatar
Łukasz Nowak committed
29
import logging
Łukasz Nowak's avatar
Łukasz Nowak committed
30
import os
Marco Mariani's avatar
Marco Mariani committed
31
import random
Łukasz Nowak's avatar
Łukasz Nowak committed
32
import shutil
33
import signal
Łukasz Nowak's avatar
Łukasz Nowak committed
34
import socket
35
import sys
36
import stat
37
import tempfile
Marco Mariani's avatar
Marco Mariani committed
38
import textwrap
39
import time
40
import unittest
Łukasz Nowak's avatar
Łukasz Nowak committed
41
import urlparse
42
import json
Marco Mariani's avatar
Marco Mariani committed
43

44
import xml_marshaller
45
from mock import patch
Łukasz Nowak's avatar
Łukasz Nowak committed
46

Marco Mariani's avatar
Marco Mariani committed
47 48 49 50
import slapos.slap.slap
import slapos.grid.utils
from slapos.grid import slapgrid
from slapos.grid.utils import md5digest
51
from slapos.grid.watchdog import Watchdog
52
from slapos.grid import SlapObject
53
from slapos.grid.SlapObject import WATCHDOG_MARK
54
import slapos.grid.SlapObject
Marco Mariani's avatar
Marco Mariani committed
55

56 57
import httmock

58

Marco Mariani's avatar
Marco Mariani committed
59 60
dummylogger = logging.getLogger()

61

62
WATCHDOG_TEMPLATE = """#!{python_path} -S
63
import sys
64 65
sys.path={sys_path}
import slapos.slap
66 67
import slapos.grid.watchdog

68 69 70 71 72 73 74 75 76 77 78
def bang(self_partition, message):
  nl = chr(10)
  with open('{watchdog_banged}', 'w') as fout:
    for key, value in vars(self_partition).items():
      fout.write('%s: %s%s' % (key, value, nl))
      if key == '_connection_helper':
        for k, v in vars(value).items():
          fout.write('   %s: %s%s' % (k, v, nl))
    fout.write(message)

slapos.slap.ComputerPartition.bang = bang
79 80 81
slapos.grid.watchdog.main()
"""

82 83 84 85
WRAPPER_CONTENT = """#!/bin/sh
touch worked &&
mkdir -p etc/run &&
echo "#!/bin/sh" > etc/run/wrapper &&
Marco Mariani's avatar
Marco Mariani committed
86
echo "while true; do echo Working; sleep 0.1; done" >> etc/run/wrapper &&
87 88 89
chmod 755 etc/run/wrapper
"""

90 91 92 93 94
DAEMON_CONTENT = """#!/bin/sh
mkdir -p etc/service &&
echo "#!/bin/sh" > etc/service/daemon &&
echo "touch launched
if [ -f ./crashed ]; then
Marco Mariani's avatar
Marco Mariani committed
95
while true; do echo Working; sleep 0.1; done
96
else
Marco Mariani's avatar
Marco Mariani committed
97
touch ./crashed; echo Failing; sleep 1; exit 111;
98 99 100 101 102
fi" >> etc/service/daemon &&
chmod 755 etc/service/daemon &&
touch worked
"""

103

104 105

class BasicMixin(object):
Łukasz Nowak's avatar
Łukasz Nowak committed
106 107
  def setUp(self):
    self._tempdir = tempfile.mkdtemp()
108 109
    self.software_root = os.path.join(self._tempdir, 'software')
    self.instance_root = os.path.join(self._tempdir, 'instance')
110 111
    if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
      del os.environ['SLAPGRID_INSTANCE_ROOT']
112 113 114
    logging.basicConfig(level=logging.DEBUG)
    self.setSlapgrid()

Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
115
  def setSlapgrid(self, develop=False):
116
    if getattr(self, 'master_url', None) is None:
117
      self.master_url = 'http://127.0.0.1:80/'
Łukasz Nowak's avatar
Łukasz Nowak committed
118 119 120
    self.computer_id = 'computer'
    self.supervisord_socket = os.path.join(self._tempdir, 'supervisord.sock')
    self.supervisord_configuration_path = os.path.join(self._tempdir,
121
                                                       'supervisord')
Łukasz Nowak's avatar
Łukasz Nowak committed
122 123
    self.usage_report_periodicity = 1
    self.buildout = None
124 125 126 127 128 129 130
    self.grid = slapgrid.Slapgrid(self.software_root,
                                  self.instance_root,
                                  self.master_url,
                                  self.computer_id,
                                  self.buildout,
                                  develop=develop,
                                  logger=logging.getLogger())
131
    # monkey patch buildout bootstrap
132

133 134
    def dummy(*args, **kw):
      pass
135

136
    slapos.grid.utils.bootstrapBuildout = dummy
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
137

138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
    SlapObject.PROGRAM_PARTITION_TEMPLATE = textwrap.dedent("""\
        [program:%(program_id)s]
        directory=%(program_directory)s
        command=%(program_command)s
        process_name=%(program_name)s
        autostart=false
        autorestart=false
        startsecs=0
        startretries=0
        exitcodes=0
        stopsignal=TERM
        stopwaitsecs=60
        stopasgroup=true
        killasgroup=true
        user=%(user_id)s
        group=%(group_id)s
        serverurl=AUTO
        redirect_stderr=true
        stdout_logfile=%(instance_path)s/.%(program_id)s.log
        stderr_logfile=%(instance_path)s/.%(program_id)s.log
        environment=USER="%(USER)s",LOGNAME="%(USER)s",HOME="%(HOME)s"
        """)

Marco Mariani's avatar
Marco Mariani committed
161
  def launchSlapgrid(self, develop=False):
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
162 163
    self.setSlapgrid(develop=develop)
    return self.grid.processComputerPartitionList()
164

Marco Mariani's avatar
Marco Mariani committed
165
  def launchSlapgridSoftware(self, develop=False):
166 167 168
    self.setSlapgrid(develop=develop)
    return self.grid.processSoftwareReleaseList()

Marco Mariani's avatar
Marco Mariani committed
169 170 171 172
  def assertLogContent(self, log_path, expected, tries=50):
    for i in range(tries):
      if expected in open(log_path).read():
        return
173
      time.sleep(0.01)
Marco Mariani's avatar
Marco Mariani committed
174 175 176 177 178 179
    self.fail('%r not found in %s' % (expected, log_path))

  def assertIsCreated(self, path, tries=50):
    for i in range(tries):
      if os.path.exists(path):
        return
180
      time.sleep(0.01)
Marco Mariani's avatar
Marco Mariani committed
181 182 183 184 185 186
    self.fail('%s should be created' % path)

  def assertIsNotCreated(self, path, tries=50):
    for i in range(tries):
      if os.path.exists(path):
        self.fail('%s should not be created' % path)
187
      time.sleep(0.01)
Marco Mariani's avatar
Marco Mariani committed
188

189 190 191 192 193 194
  def assertInstanceDirectoryListEqual(self, instance_list):
    instance_list.append('etc')
    instance_list.append('var')
    instance_list.append('supervisord.socket')
    self.assertItemsEqual(os.listdir(self.instance_root), instance_list)

Łukasz Nowak's avatar
Łukasz Nowak committed
195
  def tearDown(self):
Łukasz Nowak's avatar
Łukasz Nowak committed
196 197 198 199 200 201 202 203 204
    # XXX: Hardcoded pid, as it is not configurable in slapos
    svc = os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')
    if os.path.exists(svc):
      try:
        pid = int(open(svc).read().strip())
      except ValueError:
        pass
      else:
        os.kill(pid, signal.SIGTERM)
205
    shutil.rmtree(self._tempdir, True)
Łukasz Nowak's avatar
Łukasz Nowak committed
206

207

208
class TestRequiredOnlyPartitions(unittest.TestCase):
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
  def test_no_errors(self):
    required = ['one', 'three']
    existing = ['one', 'two', 'three']
    slapgrid.check_required_only_partitions(existing, required)

  def test_one_missing(self):
    required = ['foobar', 'two', 'one']
    existing = ['one', 'two', 'three']
    self.assertRaisesRegexp(ValueError,
                            'Unknown partition: foobar',
                            slapgrid.check_required_only_partitions,
                            existing, required)

  def test_several_missing(self):
    required = ['foobar', 'barbaz']
    existing = ['one', 'two', 'three']
    self.assertRaisesRegexp(ValueError,
                            'Unknown partitions: barbaz, foobar',
                            slapgrid.check_required_only_partitions,
                            existing, required)


231
class TestBasicSlapgridCP(BasicMixin, unittest.TestCase):
Łukasz Nowak's avatar
Łukasz Nowak committed
232 233 234 235 236 237 238
  def test_no_software_root(self):
    self.assertRaises(OSError, self.grid.processComputerPartitionList)

  def test_no_instance_root(self):
    os.mkdir(self.software_root)
    self.assertRaises(OSError, self.grid.processComputerPartitionList)

239
  @unittest.skip('which request handler here?')
Łukasz Nowak's avatar
Łukasz Nowak committed
240 241 242 243
  def test_no_master(self):
    os.mkdir(self.software_root)
    os.mkdir(self.instance_root)
    self.assertRaises(socket.error, self.grid.processComputerPartitionList)
244

245

246
class MasterMixin(BasicMixin):
247

248 249 250 251 252 253 254 255 256 257 258 259 260
  def _mock_sleep(self):
    self.fake_waiting_time = None
    self.real_sleep = time.sleep

    def mocked_sleep(secs):
      if self.fake_waiting_time is not None:
        secs = self.fake_waiting_time
      self.real_sleep(secs)

    time.sleep = mocked_sleep

  def _unmock_sleep(self):
    time.sleep = self.real_sleep
261

262
  def setUp(self):
263
    self._mock_sleep()
264
    BasicMixin.setUp(self)
265 266

  def tearDown(self):
267
    self._unmock_sleep()
268 269
    BasicMixin.tearDown(self)

270

271
class ComputerForTest(object):
272 273 274 275
  """
  Class to set up environment for tests setting instance, software
  and server response
  """
276 277 278 279 280
  def __init__(self,
               software_root,
               instance_root,
               instance_amount=1,
               software_amount=1):
281 282 283 284
    """
    Will set up instances, software and sequence
    """
    self.sequence = []
285 286 287 288
    self.instance_amount = instance_amount
    self.software_amount = software_amount
    self.software_root = software_root
    self.instance_root = instance_root
289 290 291 292 293
    self.ip_address_list = [
            ('interface1', '10.0.8.3'),
            ('interface2', '10.0.8.4'),
            ('route_interface1', '10.10.8.4')
      ]
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
294 295 296 297
    if not os.path.isdir(self.instance_root):
      os.mkdir(self.instance_root)
    if not os.path.isdir(self.software_root):
      os.mkdir(self.software_root)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
298 299
    self.setSoftwares()
    self.setInstances()
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319


  def request_handler(self, url, req):
    """
    Define _callback.
    Will register global sequence of message, sequence by partition
    and error and error message by partition
    """
    self.sequence.append(url.path)
    if req.method == 'GET':
      qs = urlparse.parse_qs(url.query)
    else:
      qs = urlparse.parse_qs(req.body)
    if (url.path == '/getFullComputerInformation'
            and 'computer_id' in qs):
      slap_computer = self.getComputer(qs['computer_id'][0])
      return {
              'status_code': 200,
              'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
              }
320 321 322 323 324 325
    elif url.path == '/getHostingSubscriptionIpList':
      ip_address_list = self.ip_address_list
      return {
              'status_code': 200,
              'content': xml_marshaller.xml_marshaller.dumps(ip_address_list)
              }
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
    if req.method == 'POST' and 'computer_partition_id' in qs:
      instance = self.instance_list[int(qs['computer_partition_id'][0])]
      instance.sequence.append(url.path)
      instance.header_list.append(req.headers)
      if url.path == '/availableComputerPartition':
        return {'status_code': 200}
      if url.path == '/startedComputerPartition':
        instance.state = 'started'
        return {'status_code': 200}
      if url.path == '/stoppedComputerPartition':
        instance.state = 'stopped'
        return {'status_code': 200}
      if url.path == '/destroyedComputerPartition':
        instance.state = 'destroyed'
        return {'status_code': 200}
      if url.path == '/softwareInstanceBang':
        return {'status_code': 200}
343 344
      if url.path == "/updateComputerPartitionRelatedInstanceList":
        return {'status_code': 200}
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
      if url.path == '/softwareInstanceError':
        instance.error_log = '\n'.join(
            [
                line
                for line in qs['error_log'][0].splitlines()
                if 'dropPrivileges' not in line
            ]
        )
        instance.error = True
        return {'status_code': 200}

    elif req.method == 'POST' and 'url' in qs:
      # XXX hardcoded to first software release!
      software = self.software_list[0]
      software.sequence.append(url.path)
      if url.path == '/buildingSoftwareRelease':
        return {'status_code': 200}
      if url.path == '/softwareReleaseError':
        software.error_log = '\n'.join(
            [
                line
                for line in qs['error_log'][0].splitlines()
                if 'dropPrivileges' not in line
            ]
        )
        software.error = True
        return {'status_code': 200}

    else:
      return {'status_code': 500}



378

379 380 381 382
  def setSoftwares(self):
    """
    Will set requested amount of software
    """
Marco Mariani's avatar
Marco Mariani committed
383
    self.software_list = [
384 385 386
        SoftwareForTest(self.software_root, name=str(i))
        for i in range(self.software_amount)
    ]
387

388
  def setInstances(self):
389 390 391
    """
    Will set requested amount of instance giving them by default first software
    """
Marco Mariani's avatar
Marco Mariani committed
392 393 394 395 396 397
    if self.software_list:
      software = self.software_list[0]
    else:
      software = None

    self.instance_list = [
398 399 400
        InstanceForTest(self.instance_root, name=str(i), software=software)
        for i in range(self.instance_amount)
    ]
401

Marco Mariani's avatar
Marco Mariani committed
402
  def getComputer(self, computer_id):
403 404 405
    """
    Will return current requested state of computer
    """
406
    slap_computer = slapos.slap.Computer(computer_id)
Marco Mariani's avatar
Marco Mariani committed
407
    slap_computer._software_release_list = [
408 409 410
        software.getSoftware(computer_id)
        for software in self.software_list
    ]
Marco Mariani's avatar
Marco Mariani committed
411
    slap_computer._computer_partition_list = [
412 413 414
        instance.getInstance(computer_id)
        for instance in self.instance_list
    ]
415 416 417 418
    return slap_computer



419
class InstanceForTest(object):
420 421 422
  """
  Class containing all needed paramaters and function to simulate instances
  """
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
423
  def __init__(self, instance_root, name, software):
424
    self.instance_root = instance_root
425
    self.software = software
426
    self.requested_state = 'stopped'
427
    self.state = None
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
428
    self.error = False
429
    self.error_log = None
430
    self.sequence = []
431
    self.header_list = []
432 433
    self.name = name
    self.partition_path = os.path.join(self.instance_root, self.name)
Marco Mariani's avatar
Marco Mariani committed
434
    os.mkdir(self.partition_path, 0o750)
435
    self.timestamp = None
436 437 438
    self.ip_list = [('interface0', '10.0.8.2')]
    self.full_ip_list = [('route_interface0', '10.10.2.3', '10.10.0.1',
                          '255.0.0.0', '10.0.0.0')]
439

440
  def getInstance(self, computer_id, ):
441 442 443
    """
    Will return current requested state of instance
    """
444
    partition = slapos.slap.ComputerPartition(computer_id, self.name)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
445
    partition._software_release_document = self.getSoftwareRelease()
446
    partition._requested_state = self.requested_state
447 448
    if getattr(self, 'filter_dict', None):
      partition._filter_dict = self.filter_dict
449 450 451
    partition._parameter_dict = {'ip_list': self.ip_list,
                                  'full_ip_list': self.full_ip_list
                                  }
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
452
    if self.software is not None:
453
      if self.timestamp is not None:
454 455
        partition._parameter_dict['timestamp'] = self.timestamp
        
456
    self.current_partition = partition
457 458
    return partition

Marco Mariani's avatar
Marco Mariani committed
459
  def getSoftwareRelease(self):
460 461 462
    """
    Return software release for Instance
    """
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
463
    if self.software is not None:
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
464
      sr = slapos.slap.SoftwareRelease()
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
465 466
      sr._software_release = self.software.name
      return sr
467 468
    else:
      return None
469

Marco Mariani's avatar
Marco Mariani committed
470
  def setPromise(self, promise_name, promise_content):
471 472 473 474 475 476
    """
    This function will set promise and return its path
    """
    promise_path = os.path.join(self.partition_path, 'etc', 'promise')
    if not os.path.isdir(promise_path):
      os.makedirs(promise_path)
477
    promise = os.path.join(promise_path, promise_name)
478
    open(promise, 'w').write(promise_content)
Marco Mariani's avatar
Marco Mariani committed
479
    os.chmod(promise, 0o777)
480

481 482 483 484 485
  def setCertificate(self, certificate_repository_path):
    if not os.path.exists(certificate_repository_path):
      os.mkdir(certificate_repository_path)
    self.cert_file = os.path.join(certificate_repository_path,
                                  "%s.crt" % self.name)
Marco Mariani's avatar
Marco Mariani committed
486
    self.certificate = str(random.random())
487 488
    open(self.cert_file, 'w').write(self.certificate)
    self.key_file = os.path.join(certificate_repository_path,
489
                                 '%s.key' % self.name)
Marco Mariani's avatar
Marco Mariani committed
490
    self.key = str(random.random())
491
    open(self.key_file, 'w').write(self.key)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
492

493

494
class SoftwareForTest(object):
495 496 497 498
  """
  Class to prepare and simulate software.
  each instance has a sotfware attributed
  """
499
  def __init__(self, software_root, name=''):
500 501 502
    """
    Will set file and variable for software
    """
503 504 505
    self.software_root = software_root
    self.name = 'http://sr%s/' % name
    self.sequence = []
Marco Mariani's avatar
Marco Mariani committed
506
    self.software_hash = md5digest(self.name)
507
    self.srdir = os.path.join(self.software_root, self.software_hash)
508
    self.requested_state = 'available'
509
    os.mkdir(self.srdir)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
510
    self.setTemplateCfg()
511 512
    self.srbindir = os.path.join(self.srdir, 'bin')
    os.mkdir(self.srbindir)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
513
    self.setBuildout()
514

Marco Mariani's avatar
Marco Mariani committed
515
  def getSoftware(self, computer_id):
516 517 518 519 520 521 522
    """
    Will return current requested state of software
    """
    software = slapos.slap.SoftwareRelease(self.name, computer_id)
    software._requested_state = self.requested_state
    return software

Marco Mariani's avatar
Marco Mariani committed
523
  def setTemplateCfg(self, template="""[buildout]"""):
524 525 526
    """
    Set template.cfg
    """
527 528
    open(os.path.join(self.srdir, 'template.cfg'), 'w').write(template)

Marco Mariani's avatar
Marco Mariani committed
529
  def setBuildout(self, buildout="""#!/bin/sh
530
touch worked"""):
531 532 533
    """
    Set a buildout exec in bin
    """
534
    open(os.path.join(self.srbindir, 'buildout'), 'w').write(buildout)
Marco Mariani's avatar
Marco Mariani committed
535
    os.chmod(os.path.join(self.srbindir, 'buildout'), 0o755)
536

Marco Mariani's avatar
Marco Mariani committed
537
  def setPeriodicity(self, periodicity):
538 539 540
    """
    Set a periodicity file
    """
541 542
    with open(os.path.join(self.srdir, 'periodicity'), 'w') as fout:
      fout.write(str(periodicity))
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
543 544


545
class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
546

547
  def test_nothing_to_do(self):
548 549 550
    computer = ComputerForTest(self.software_root, self.instance_root, 0, 0)
    with httmock.HTTMock(computer.request_handler):
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
551
      self.assertInstanceDirectoryListEqual([])
552 553 554
      self.assertItemsEqual(os.listdir(self.software_root), [])
      st = os.stat(os.path.join(self.instance_root, 'var'))
      self.assertEquals(stat.S_IMODE(st.st_mode), 0o755)
555 556

  def test_one_partition(self):
Marco Mariani's avatar
Marco Mariani committed
557
    computer = ComputerForTest(self.software_root, self.instance_root)
558 559 560
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
561
      self.assertInstanceDirectoryListEqual(['0'])
562 563 564 565 566 567 568
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                    'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/stoppedComputerPartition'])
569 570 571 572 573 574

  def test_one_partition_instance_cfg(self):
    """
    Check that slapgrid processes instance is profile is not named
    "template.cfg" but "instance.cfg".
    """
Marco Mariani's avatar
Marco Mariani committed
575
    computer = ComputerForTest(self.software_root, self.instance_root)
576 577 578
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
579
      self.assertInstanceDirectoryListEqual(['0'])
580 581 582 583 584 585 586
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', 'buildout.cfg',
                                                    'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/stoppedComputerPartition'])
587

588 589
  def test_one_free_partition(self):
    """
590
    Test if slapgrid cp does not process "free" partition
591
    """
Marco Mariani's avatar
Marco Mariani committed
592 593 594
    computer = ComputerForTest(self.software_root,
                               self.instance_root,
                               software_amount=0)
595 596 597 598
    with httmock.HTTMock(computer.request_handler):
      partition = computer.instance_list[0]
      partition.requested_state = 'destroyed'
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
599
      self.assertInstanceDirectoryListEqual(['0'])
600 601 602
      self.assertItemsEqual(os.listdir(partition.partition_path), [])
      self.assertItemsEqual(os.listdir(self.software_root), [])
      self.assertEqual(partition.sequence, [])
603

604
  def test_one_partition_started(self):
Marco Mariani's avatar
Marco Mariani committed
605
    computer = ComputerForTest(self.software_root, self.instance_root)
606 607 608 609 610
    with httmock.HTTMock(computer.request_handler):
      partition = computer.instance_list[0]
      partition.requested_state = 'started'
      partition.software.setBuildout(WRAPPER_CONTENT)
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
611
      self.assertInstanceDirectoryListEqual(['0'])
612 613 614 615 616 617 618 619 620 621
      self.assertItemsEqual(os.listdir(partition.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/startedComputerPartition'])
      self.assertEqual(partition.state, 'started')
622

623
  def test_one_partition_started_stopped(self):
Marco Mariani's avatar
Marco Mariani committed
624
    computer = ComputerForTest(self.software_root, self.instance_root)
625 626
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
627

628 629
      instance.requested_state = 'started'
      instance.software.setBuildout("""#!/bin/sh
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
touch worked &&
mkdir -p etc/run &&
(
cat <<'HEREDOC'
#!%(python)s
import signal
def handler(signum, frame):
  print 'Signal handler called with signal', signum
  raise SystemExit
signal.signal(signal.SIGTERM, handler)

while True:
  print "Working"
HEREDOC
)> etc/run/wrapper &&
chmod 755 etc/run/wrapper
Marco Mariani's avatar
Marco Mariani committed
646
""" % {'python': sys.executable})
647
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
648
      self.assertInstanceDirectoryListEqual(['0'])
649 650 651 652 653 654 655 656 657 658 659 660 661 662
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/startedComputerPartition'])
      self.assertEqual(instance.state, 'started')

      computer.sequence = []
      instance.requested_state = 'stopped'
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
663
      self.assertInstanceDirectoryListEqual(['0'])
664 665 666 667 668
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertLogContent(wrapper_log, 'Signal handler called with signal 15')
      self.assertEqual(computer.sequence,
669
                       ['/getHateoasUrl', '/getFullComputerInformation', '/availableComputerPartition',
670 671
                        '/stoppedComputerPartition'])
      self.assertEqual(instance.state, 'stopped')
672

673 674 675 676 677 678
  def test_one_broken_partition_stopped(self):
    """
    Check that, for, an already started instance if stop is requested,
    processes will be stopped even if instance is broken (buildout fails
    to run) but status is still started.
    """
Marco Mariani's avatar
Marco Mariani committed
679
    computer = ComputerForTest(self.software_root, self.instance_root)
680 681
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
682

683 684
      instance.requested_state = 'started'
      instance.software.setBuildout("""#!/bin/sh
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700
touch worked &&
mkdir -p etc/run &&
(
cat <<'HEREDOC'
#!%(python)s
import signal
def handler(signum, frame):
  print 'Signal handler called with signal', signum
  raise SystemExit
signal.signal(signal.SIGTERM, handler)

while True:
  print "Working"
HEREDOC
)> etc/run/wrapper &&
chmod 755 etc/run/wrapper
Marco Mariani's avatar
Marco Mariani committed
701
""" % {'python': sys.executable})
702
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
703
      self.assertInstanceDirectoryListEqual(['0'])
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/startedComputerPartition'])
      self.assertEqual(instance.state, 'started')

      computer.sequence = []
      instance.requested_state = 'stopped'
      instance.software.setBuildout("""#!/bin/sh
719 720
exit 1
""")
721
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_FAIL)
722
      self.assertInstanceDirectoryListEqual(['0'])
723 724 725 726 727
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertLogContent(wrapper_log, 'Signal handler called with signal 15')
      self.assertEqual(computer.sequence,
728
                       ['/getHateoasUrl', '/getFullComputerInformation',
729 730
                        '/softwareInstanceError'])
      self.assertEqual(instance.state, 'started')
731

732
  def test_one_partition_stopped_started(self):
Marco Mariani's avatar
Marco Mariani committed
733
    computer = ComputerForTest(self.software_root, self.instance_root)
734 735 736 737 738 739
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'stopped'
      instance.software.setBuildout(WRAPPER_CONTENT)
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)

740
      self.assertInstanceDirectoryListEqual(['0'])
741 742 743 744 745 746 747 748 749 750 751 752 753
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', 'buildout.cfg', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation', '/availableComputerPartition',
                        '/stoppedComputerPartition'])
      self.assertEqual('stopped', instance.state)

      instance.requested_state = 'started'
      computer.sequence = []
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
754
      self.assertInstanceDirectoryListEqual(['0'])
755 756 757 758 759 760 761 762 763
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.0_wrapper.log', 'etc',
                             'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertEqual(computer.sequence,
764
                       ['/getHateoasUrl', '/getFullComputerInformation', '/availableComputerPartition',
765 766
                        '/startedComputerPartition'])
      self.assertEqual('started', instance.state)
767

768 769 770 771 772 773
  def test_one_partition_destroyed(self):
    """
    Test that an existing partition with "destroyed" status will only be
    stopped by slapgrid-cp, not processed
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
774 775 776
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'destroyed'
Marco Mariani's avatar
Marco Mariani committed
777

778 779 780
      dummy_file_name = 'dummy_file'
      with open(os.path.join(instance.partition_path, dummy_file_name), 'w') as dummy_file:
          dummy_file.write('dummy')
781

782
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
783

784
      self.assertInstanceDirectoryListEqual(['0'])
785 786 787 788 789 790 791
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition), ['.slapgrid', dummy_file_name])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation',
                        '/stoppedComputerPartition'])
      self.assertEqual('stopped', instance.state)
792

793

794
class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
795

796 797 798 799 800
  def setUp(self):
    MasterMixin.setUp(self)
    # Prepare watchdog
    self.watchdog_banged = os.path.join(self._tempdir, 'watchdog_banged')
    watchdog_path = os.path.join(self._tempdir, 'watchdog')
801 802 803 804 805
    open(watchdog_path, 'w').write(WATCHDOG_TEMPLATE.format(
        python_path=sys.executable,
        sys_path=sys.path,
        watchdog_banged=self.watchdog_banged
    ))
Marco Mariani's avatar
Marco Mariani committed
806
    os.chmod(watchdog_path, 0o755)
807 808 809
    self.grid.watchdog_path = watchdog_path
    slapos.grid.slapgrid.WATCHDOG_PATH = watchdog_path

810 811 812 813 814 815 816 817 818 819 820 821
  def test_one_failing_daemon_in_service_will_bang_with_watchdog(self):
    """
    Check that a failing service watched by watchdog trigger bang
    1.Prepare computer and set a service named daemon in etc/service
       (to be watched by watchdog). This daemon will fail.
    2.Prepare file for supervisord to call watchdog
       -Set sys.path
       -Monkeypatch computer partition bang
    3.Check damemon is launched
    4.Wait for it to fail
    5.Wait for file generated by monkeypacthed bang to appear
    """
Cédric de Saint Martin's avatar
PEP8  
Cédric de Saint Martin committed
822
    computer = ComputerForTest(self.software_root, self.instance_root)
823 824 825 826 827 828
    with httmock.HTTMock(computer.request_handler):
      partition = computer.instance_list[0]
      partition.requested_state = 'started'
      partition.software.setBuildout(DAEMON_CONTENT)

      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
829
      self.assertInstanceDirectoryListEqual(['0'])
830 831 832 833 834 835 836
      self.assertItemsEqual(os.listdir(partition.partition_path),
                            ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      daemon_log = os.path.join(partition.partition_path, '.0_daemon.log')
      self.assertLogContent(daemon_log, 'Failing')
      self.assertIsCreated(self.watchdog_banged)
      self.assertIn('daemon', open(self.watchdog_banged).read())
837 838 839

  def test_one_failing_daemon_in_run_will_not_bang_with_watchdog(self):
    """
840
    Check that a failing service watched by watchdog does not trigger bang
841 842 843 844 845 846 847 848 849
    1.Prepare computer and set a service named daemon in etc/run
       (not watched by watchdog). This daemon will fail.
    2.Prepare file for supervisord to call watchdog
       -Set sys.path
       -Monkeypatch computer partition bang
    3.Check damemon is launched
    4.Wait for it to fail
    5.Check that file generated by monkeypacthed bang do not appear
    """
Marco Mariani's avatar
Marco Mariani committed
850
    computer = ComputerForTest(self.software_root, self.instance_root)
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873
    with httmock.HTTMock(computer.request_handler):
      partition = computer.instance_list[0]
      partition.requested_state = 'started'

      # Content of run wrapper
      WRAPPER_CONTENT = textwrap.dedent("""#!/bin/sh
          touch ./launched
          touch ./crashed
          echo Failing
          sleep 1
          exit 111
      """)

      BUILDOUT_RUN_CONTENT = textwrap.dedent("""#!/bin/sh
          mkdir -p etc/run &&
          echo "%s" >> etc/run/daemon &&
          chmod 755 etc/run/daemon &&
          touch worked
          """ % WRAPPER_CONTENT)

      partition.software.setBuildout(BUILDOUT_RUN_CONTENT)

      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
874
      self.assertInstanceDirectoryListEqual(['0'])
875 876 877 878 879 880
      self.assertItemsEqual(os.listdir(partition.partition_path),
                            ['.slapgrid', '.0_daemon.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      daemon_log = os.path.join(partition.partition_path, '.0_daemon.log')
      self.assertLogContent(daemon_log, 'Failing')
      self.assertIsNotCreated(self.watchdog_banged)
881 882 883 884 885

  def test_watched_by_watchdog_bang(self):
    """
    Test that a process going to fatal or exited mode in supervisord
    is banged if watched by watchdog
886
    Certificates used for the bang are also checked
887 888
    (ie: watchdog id in process name)
    """
Marco Mariani's avatar
Marco Mariani committed
889
    computer = ComputerForTest(self.software_root, self.instance_root)
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      certificate_repository_path = os.path.join(self._tempdir, 'partition_pki')
      instance.setCertificate(certificate_repository_path)

      watchdog = Watchdog(
          master_url='https://127.0.0.1/',
          computer_id=self.computer_id,
          certificate_repository_path=certificate_repository_path
      )
      for event in watchdog.process_state_events:
        instance.sequence = []
        instance.header_list = []
        headers = {'eventname': event}
        payload = 'processname:%s groupname:%s from_state:RUNNING' % (
            'daemon' + WATCHDOG_MARK, instance.name)
        watchdog.handle_event(headers, payload)
        self.assertEqual(instance.sequence, ['/softwareInstanceBang'])
908 909 910 911 912 913

  def test_unwanted_events_will_not_bang(self):
    """
    Test that a process going to a mode not watched by watchdog
    in supervisord is not banged if watched by watchdog
    """
Marco Mariani's avatar
Marco Mariani committed
914
    computer = ComputerForTest(self.software_root, self.instance_root)
915 916
    instance = computer.instance_list[0]

917 918 919 920 921
    watchdog = Watchdog(
        master_url=self.master_url,
        computer_id=self.computer_id,
        certificate_repository_path=None
    )
922 923 924
    for event in ['EVENT', 'PROCESS_STATE', 'PROCESS_STATE_RUNNING',
                  'PROCESS_STATE_BACKOFF', 'PROCESS_STATE_STOPPED']:
      computer.sequence = []
Marco Mariani's avatar
Marco Mariani committed
925
      headers = {'eventname': event}
926
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
927
          'daemon' + WATCHDOG_MARK, instance.name)
Marco Mariani's avatar
Marco Mariani committed
928 929
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, [])
930 931 932 933 934 935 936

  def test_not_watched_by_watchdog_do_not_bang(self):
    """
    Test that a process going to fatal or exited mode in supervisord
    is not banged if not watched by watchdog
    (ie: no watchdog id in process name)
    """
Marco Mariani's avatar
Marco Mariani committed
937
    computer = ComputerForTest(self.software_root, self.instance_root)
938 939
    instance = computer.instance_list[0]

940 941 942 943 944
    watchdog = Watchdog(
        master_url=self.master_url,
        computer_id=self.computer_id,
        certificate_repository_path=None
    )
945 946
    for event in watchdog.process_state_events:
      computer.sequence = []
Marco Mariani's avatar
Marco Mariani committed
947
      headers = {'eventname': event}
948
      payload = "processname:%s groupname:%s from_state:RUNNING"\
Marco Mariani's avatar
Marco Mariani committed
949 950 951
          % ('daemon', instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(computer.sequence, [])
952

953 954 955 956 957 958 959
  def test_watchdog_create_bang_file_after_bang(self):
    """
    For a partition that has been successfully deployed (thus .timestamp file
    existing), check that bang file is created and contains the timestamp of
    .timestamp file.
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      certificate_repository_path = os.path.join(self._tempdir, 'partition_pki')
      instance.setCertificate(certificate_repository_path)
      partition = os.path.join(self.instance_root, '0')
      timestamp_content = '1234'
      timestamp_file = open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_TIMESTAMP_FILENAME), 'w')
      timestamp_file.write(timestamp_content)
      timestamp_file.close()

      watchdog = Watchdog(
          master_url='https://127.0.0.1/',
          computer_id=self.computer_id,
          certificate_repository_path=certificate_repository_path,
          instance_root_path=self.instance_root
      )
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, ['/softwareInstanceBang'])
984

985
      self.assertEqual(open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME)).read(), timestamp_content)
986 987 988 989 990 991 992 993 994 995


  def test_watchdog_ignore_bang_if_partition_not_deployed(self):
    """
    For a partition that has never been successfully deployed (buildout is
    failing, promise is not passing, etc), test that bang is ignored.

    Practically speaking, .timestamp file in the partition does not exsit.
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      certificate_repository_path = os.path.join(self._tempdir, 'partition_pki')
      instance.setCertificate(certificate_repository_path)
      partition = os.path.join(self.instance_root, '0')
      timestamp_content = '1234'

      watchdog = Watchdog(
          master_url='https://127.0.0.1/',
          computer_id=self.computer_id,
          certificate_repository_path=certificate_repository_path,
          instance_root_path=self.instance_root
      )
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, ['/softwareInstanceBang'])
1017

1018
      self.assertNotEqual(open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME)).read(), timestamp_content)
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028


  def test_watchdog_bang_only_once_if_partition_never_deployed(self):
    """
    For a partition that has been never successfully deployed (promises are not passing,
    etc), test that:
     * First bang is transmitted
     * subsequent bangs are ignored until a deployment is successful.
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      certificate_repository_path = os.path.join(self._tempdir, 'partition_pki')
      instance.setCertificate(certificate_repository_path)
      partition = os.path.join(self.instance_root, '0')

      watchdog = Watchdog(
          master_url='https://127.0.0.1/',
          computer_id=self.computer_id,
          certificate_repository_path=certificate_repository_path,
          instance_root_path=self.instance_root
      )
      # First bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, ['/softwareInstanceBang'])
1050

1051 1052 1053 1054 1055 1056 1057 1058 1059
      # Second bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, [])
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078


  def test_watchdog_bang_only_once_if_timestamp_did_not_change(self):
    """
    For a partition that has been successfully deployed (promises are passing,
    etc), test that:
     * First bang is transmitted
     * subsequent bangs are ignored until a new deployment is successful.
    Scenario:
     * slapgrid successfully deploys a partition
     * A process crashes, watchdog calls bang
     * Another deployment (run of slapgrid) is done, but not successful (
       promise is failing)
     * The process crashes again, but watchdog ignores it
     * Yet another deployment is done, and it is successful
     * The process crashes again, watchdog calls bang
     * The process crashes again, watchdog ignroes it
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      certificate_repository_path = os.path.join(self._tempdir, 'partition_pki')
      instance.setCertificate(certificate_repository_path)
      partition = os.path.join(self.instance_root, '0')
      timestamp_content = '1234'
      timestamp_file = open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_TIMESTAMP_FILENAME), 'w')
      timestamp_file.write(timestamp_content)
      timestamp_file.close()

      watchdog = Watchdog(
          master_url='https://127.0.0.1/',
          computer_id=self.computer_id,
          certificate_repository_path=certificate_repository_path,
          instance_root_path=self.instance_root
      )
      # First bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, ['/softwareInstanceBang'])
1104

1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
      self.assertEqual(open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME)).read(), timestamp_content)

      # Second bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, [])

      # Second successful deployment
      timestamp_content = '12345'
      timestamp_file = open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_TIMESTAMP_FILENAME), 'w')
      timestamp_file.write(timestamp_content)
      timestamp_file.close()

      # Third bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, ['/softwareInstanceBang'])

      self.assertEqual(open(os.path.join(partition, slapos.grid.slapgrid.COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME)).read(), timestamp_content)

      # Fourth bang
      event = watchdog.process_state_events[0]
      instance.sequence = []
      instance.header_list = []
      headers = {'eventname': event}
      payload = 'processname:%s groupname:%s from_state:RUNNING' % (
          'daemon' + WATCHDOG_MARK, instance.name)
      watchdog.handle_event(headers, payload)
      self.assertEqual(instance.sequence, [])
1144

1145
class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
1146

1147
  def test_partition_timestamp(self):
Marco Mariani's avatar
Marco Mariani committed
1148
    computer = ComputerForTest(self.software_root, self.instance_root)
1149 1150 1151 1152 1153 1154
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
      instance.timestamp = timestamp

      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
1155
      self.assertInstanceDirectoryListEqual(['0'])
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      timestamp_path = os.path.join(instance.partition_path, '.timestamp')
      self.setSlapgrid()
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertIn(timestamp, open(timestamp_path).read())
      self.assertEqual(instance.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
1166

1167
  def test_partition_timestamp_develop(self):
Marco Mariani's avatar
Marco Mariani committed
1168
    computer = ComputerForTest(self.software_root, self.instance_root)
1169 1170 1171 1172
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
      instance.timestamp = timestamp
1173

1174
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
1175
      self.assertInstanceDirectoryListEqual(['0'])
1176 1177 1178 1179 1180
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                             'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
1181

1182 1183 1184
      self.assertEqual(self.launchSlapgrid(develop=True),
                       slapgrid.SLAPGRID_SUCCESS)
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
1185

1186 1187 1188
      self.assertEqual(instance.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition',
                        '/availableComputerPartition', '/stoppedComputerPartition'])
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
1189 1190

  def test_partition_old_timestamp(self):
Marco Mariani's avatar
Marco Mariani committed
1191
    computer = ComputerForTest(self.software_root, self.instance_root)
1192 1193 1194 1195 1196 1197
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
      instance.timestamp = timestamp

      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
1198
      self.assertInstanceDirectoryListEqual(['0'])
1199 1200 1201 1202 1203 1204 1205 1206
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      instance.timestamp = str(int(timestamp) - 1)
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
      self.assertEqual(instance.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
1207 1208

  def test_partition_timestamp_new_timestamp(self):
Marco Mariani's avatar
Marco Mariani committed
1209
    computer = ComputerForTest(self.software_root, self.instance_root)
1210 1211 1212 1213 1214 1215
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
      instance.timestamp = timestamp

      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
1216
      self.assertInstanceDirectoryListEqual(['0'])
1217 1218 1219 1220 1221 1222 1223 1224
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      instance.timestamp = str(int(timestamp) + 1)
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
      self.assertEqual(computer.sequence,
1225 1226 1227 1228
                       ['/getHateoasUrl',
                        '/getFullComputerInformation', '/availableComputerPartition',
                        '/stoppedComputerPartition',
                        '/getHateoasUrl', '/getFullComputerInformation',
1229
                        '/availableComputerPartition', '/stoppedComputerPartition',
1230
                        '/getHateoasUrl',
1231
                        '/getFullComputerInformation'])
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
1232 1233

  def test_partition_timestamp_no_timestamp(self):
Marco Mariani's avatar
Marco Mariani committed
1234
    computer = ComputerForTest(self.software_root, self.instance_root)
1235 1236 1237 1238 1239 1240
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
      instance.timestamp = timestamp

      self.launchSlapgrid()
1241
      self.assertInstanceDirectoryListEqual(['0'])
1242 1243 1244 1245 1246 1247 1248 1249
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      instance.timestamp = None
      self.launchSlapgrid()
      self.assertEqual(computer.sequence,
1250 1251 1252 1253
                       ['/getHateoasUrl',
                        '/getFullComputerInformation', '/availableComputerPartition',
                        '/stoppedComputerPartition',
                        '/getHateoasUrl', '/getFullComputerInformation',
1254
                        '/availableComputerPartition', '/stoppedComputerPartition'])
1255

1256 1257 1258 1259 1260
  def test_partition_periodicity_remove_timestamp(self):
    """
    Check that if periodicity forces run of buildout for a partition, it
    removes the .timestamp file.
    """
Marco Mariani's avatar
Marco Mariani committed
1261
    computer = ComputerForTest(self.software_root, self.instance_root)
1262 1263 1264
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      timestamp = str(int(time.time()))
1265

1266 1267 1268
      instance.timestamp = timestamp
      instance.requested_state = 'started'
      instance.software.setPeriodicity(1)
1269

1270 1271 1272 1273 1274
      self.launchSlapgrid()
      partition = os.path.join(self.instance_root, '0')
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                             'software_release', 'worked', '.slapos-retention-lock-delay'])
1275

1276 1277 1278 1279
      time.sleep(2)
      # dummify install() so that it doesn't actually do anything so that it
      # doesn't recreate .timestamp.
      instance.install = lambda: None
1280

1281 1282 1283 1284
      self.launchSlapgrid()
      self.assertItemsEqual(os.listdir(partition),
                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                             'software_release', 'worked', '.slapos-retention-lock-delay'])
1285 1286 1287 1288 1289 1290

  def test_one_partition_periodicity_from_file_does_not_disturb_others(self):
    """
    If time between last processing of instance and now is superior
    to periodicity then instance should be proceed
    1. We set a wanted maximum_periodicity in periodicity file in
1291
        in one software release directory and not the other one
1292 1293 1294
    2. We process computer partition and check if wanted_periodicity was
        used as maximum_periodicty
    3. We wait for a time superior to wanted_periodicty
1295 1296
    4. We launch processComputerPartition and check that partition using
        software with periodicity was runned and not the other
1297 1298
    5. We check that modification time of .timestamp was modified
    """
Marco Mariani's avatar
Marco Mariani committed
1299
    computer = ComputerForTest(self.software_root, self.instance_root, 20, 20)
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      timestamp = str(int(time.time() - 5))
      instance0.timestamp = timestamp
      instance0.requested_state = 'started'
      for instance in computer.instance_list[1:]:
        instance.software = \
            computer.software_list[computer.instance_list.index(instance)]
        instance.timestamp = timestamp

      wanted_periodicity = 1
      instance0.software.setPeriodicity(wanted_periodicity)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
1312

1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
      self.launchSlapgrid()
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
      last_runtime = os.path.getmtime(
          os.path.join(instance0.partition_path, '.timestamp'))
      time.sleep(wanted_periodicity + 1)
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      time.sleep(1)
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/availableComputerPartition', '/startedComputerPartition',
                        '/availableComputerPartition', '/startedComputerPartition',
                        ])
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      self.assertGreater(
          os.path.getmtime(os.path.join(instance0.partition_path, '.timestamp')),
          last_runtime)
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
1334 1335 1336

  def test_one_partition_stopped_is_not_processed_after_periodicity(self):
    """
1337
    Check that periodicity forces processing a partition even if it is not
1338 1339
    started.
    """
Marco Mariani's avatar
Marco Mariani committed
1340
    computer = ComputerForTest(self.software_root, self.instance_root, 20, 20)
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      timestamp = str(int(time.time() - 5))
      instance0.timestamp = timestamp
      for instance in computer.instance_list[1:]:
        instance.software = \
            computer.software_list[computer.instance_list.index(instance)]
        instance.timestamp = timestamp

      wanted_periodicity = 1
      instance0.software.setPeriodicity(wanted_periodicity)
1352

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
      self.launchSlapgrid()
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
      last_runtime = os.path.getmtime(
          os.path.join(instance0.partition_path, '.timestamp'))
      time.sleep(wanted_periodicity + 1)
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      time.sleep(1)
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition',
                        '/availableComputerPartition', '/stoppedComputerPartition'])
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      self.assertNotEqual(os.path.getmtime(os.path.join(instance0.partition_path,
                                                        '.timestamp')),
                          last_runtime)
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
1373

1374 1375
  def test_one_partition_destroyed_is_not_processed_after_periodicity(self):
    """
1376
    Check that periodicity forces processing a partition even if it is not
1377 1378
    started.
    """
Marco Mariani's avatar
Marco Mariani committed
1379
    computer = ComputerForTest(self.software_root, self.instance_root, 20, 20)
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      timestamp = str(int(time.time() - 5))
      instance0.timestamp = timestamp
      instance0.requested_state = 'stopped'
      for instance in computer.instance_list[1:]:
        instance.software = \
            computer.software_list[computer.instance_list.index(instance)]
        instance.timestamp = timestamp

      wanted_periodicity = 1
      instance0.software.setPeriodicity(wanted_periodicity)
1392

1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
      self.launchSlapgrid()
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
      last_runtime = os.path.getmtime(
          os.path.join(instance0.partition_path, '.timestamp'))
      time.sleep(wanted_periodicity + 1)
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      time.sleep(1)
      instance0.requested_state = 'destroyed'
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition',
                                                       '/stoppedComputerPartition'])
      for instance in computer.instance_list[1:]:
        self.assertEqual(instance.sequence,
                         ['/availableComputerPartition', '/stoppedComputerPartition'])
      self.assertNotEqual(os.path.getmtime(os.path.join(instance0.partition_path,
                                                        '.timestamp')),
                          last_runtime)
      self.assertNotEqual(wanted_periodicity, self.grid.maximum_periodicity)
1414

1415 1416 1417 1418
  def test_one_partition_is_never_processed_when_periodicity_is_negative(self):
    """
    Checks that a partition is not processed when
    its periodicity is negative
1419 1420 1421 1422 1423
    1. We setup one instance and set periodicity at -1
    2. We mock the install method from slapos.grid.slapgrid.Partition
    3. We launch slapgrid once so that .timestamp file is created and check that install method is
    indeed called (through mocked_method.called
    4. We launch slapgrid anew and check that install as not been called again
1424 1425 1426
    """

    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
    with httmock.HTTMock(computer.request_handler):
      timestamp = str(int(time.time()))
      instance = computer.instance_list[0]
      instance.software.setPeriodicity(-1)
      instance.timestamp = timestamp
      with patch.object(slapos.grid.slapgrid.Partition, 'install', return_value=None) as mock_method:
        self.launchSlapgrid()
        self.assertTrue(mock_method.called)
        self.launchSlapgrid()
        self.assertEqual(mock_method.call_count, 1)
1437 1438 1439 1440 1441

  def test_one_partition_is_always_processed_when_periodicity_is_zero(self):
    """
    Checks that a partition is always processed when
    its periodicity is 0
1442 1443 1444 1445 1446
    1. We setup one instance and set periodicity at 0
    2. We mock the install method from slapos.grid.slapgrid.Partition
    3. We launch slapgrid once so that .timestamp file is created
    4. We launch slapgrid anew and check that install has been called twice (one time because of the
    new setup and one time because of periodicity = 0)
1447 1448 1449
    """

    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
1450 1451 1452 1453 1454 1455 1456 1457 1458
    with httmock.HTTMock(computer.request_handler):
      timestamp = str(int(time.time()))
      instance = computer.instance_list[0]
      instance.software.setPeriodicity(0)
      instance.timestamp = timestamp
      with patch.object(slapos.grid.slapgrid.Partition, 'install', return_value=None) as mock_method:
        self.launchSlapgrid()
        self.launchSlapgrid()
        self.assertEqual(mock_method.call_count, 2)
1459

1460 1461 1462 1463 1464
  def test_one_partition_buildout_fail_does_not_disturb_others(self):
    """
    1. We set up two instance one using a corrupted buildout
    2. One will fail but the other one will be processed correctly
    """
Marco Mariani's avatar
Marco Mariani committed
1465
    computer = ComputerForTest(self.software_root, self.instance_root, 2, 2)
1466 1467 1468 1469 1470
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      instance1 = computer.instance_list[1]
      instance1.software = computer.software_list[1]
      instance0.software.setBuildout("""#!/bin/sh
1471
exit 42""")
1472 1473 1474 1475 1476
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/softwareInstanceError'])
      self.assertEqual(instance1.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
1477 1478 1479 1480 1481 1482

  def test_one_partition_lacking_software_path_does_not_disturb_others(self):
    """
    1. We set up two instance but remove software path of one
    2. One will fail but the other one will be processed correctly
    """
Marco Mariani's avatar
Marco Mariani committed
1483
    computer = ComputerForTest(self.software_root, self.instance_root, 2, 2)
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      instance1 = computer.instance_list[1]
      instance1.software = computer.software_list[1]
      shutil.rmtree(instance0.software.srdir)
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/softwareInstanceError'])
      self.assertEqual(instance1.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
1494 1495 1496 1497 1498 1499

  def test_one_partition_lacking_software_bin_path_does_not_disturb_others(self):
    """
    1. We set up two instance but remove software bin path of one
    2. One will fail but the other one will be processed correctly
    """
Marco Mariani's avatar
Marco Mariani committed
1500
    computer = ComputerForTest(self.software_root, self.instance_root, 2, 2)
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      instance1 = computer.instance_list[1]
      instance1.software = computer.software_list[1]
      shutil.rmtree(instance0.software.srbindir)
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/softwareInstanceError'])
      self.assertEqual(instance1.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
1511 1512 1513

  def test_one_partition_lacking_path_does_not_disturb_others(self):
    """
1514
    1. We set up two instances but remove path of one
1515 1516
    2. One will fail but the other one will be processed correctly
    """
Marco Mariani's avatar
Marco Mariani committed
1517
    computer = ComputerForTest(self.software_root, self.instance_root, 2, 2)
1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
    with httmock.HTTMock(computer.request_handler):
      instance0 = computer.instance_list[0]
      instance1 = computer.instance_list[1]
      instance1.software = computer.software_list[1]
      shutil.rmtree(instance0.partition_path)
      self.launchSlapgrid()
      self.assertEqual(instance0.sequence,
                       ['/softwareInstanceError'])
      self.assertEqual(instance1.sequence,
                       ['/availableComputerPartition', '/stoppedComputerPartition'])
1528

1529 1530 1531 1532 1533 1534
  def test_one_partition_buildout_fail_is_correctly_logged(self):
    """
    1. We set up an instance using a corrupted buildout
    2. It will fail, make sure that whole log is sent to master
    """
    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
1535 1536
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
1537

1538 1539 1540
      line1 = "Nerdy kitten: Can I haz a process crash?"
      line2 = "Cedric: Sure, here it is."
      instance.software.setBuildout("""#!/bin/sh
1541
echo %s; echo %s; exit 42""" % (line1, line2))
1542 1543 1544 1545 1546 1547
      self.launchSlapgrid()
      self.assertEqual(instance.sequence, ['/softwareInstanceError'])
      # We don't care of actual formatting, we just want to have full log
      self.assertIn(line1, instance.error_log)
      self.assertIn(line2, instance.error_log)
      self.assertIn('Failed to run buildout', instance.error_log)
1548

1549

1550
class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
1551 1552 1553 1554 1555 1556 1557 1558
  """
  Test suite about slapgrid-ur
  """

  def test_slapgrid_destroys_instance_to_be_destroyed(self):
    """
    Test than an instance in "destroyed" state is correctly destroyed
    """
Marco Mariani's avatar
Marco Mariani committed
1559
    computer = ComputerForTest(self.software_root, self.instance_root)
1560 1561 1562 1563 1564
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      instance.software.setBuildout(WRAPPER_CONTENT)
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
1565
      self.assertInstanceDirectoryListEqual(['0'])
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation',
                        '/availableComputerPartition',
                        '/startedComputerPartition'])
      self.assertEqual(instance.state, 'started')

      # Then destroy the instance
      computer.sequence = []
      instance.requested_state = 'destroyed'
      self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
      # Assert partition directory is empty
1583
      self.assertInstanceDirectoryListEqual(['0'])
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
      self.assertItemsEqual(os.listdir(instance.partition_path), [])
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      # Assert supervisor stopped process
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertIsNotCreated(wrapper_log)

      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation',
                        '/stoppedComputerPartition',
                        '/destroyedComputerPartition'])
      self.assertEqual(instance.state, 'destroyed')
1596

1597
  def test_partition_list_is_complete_if_empty_destroyed_partition(self):
1598
    """
1599 1600 1601 1602 1603 1604
    Test that an empty partition with destroyed state but with SR informations
    Is correctly destroyed
    Axiom: each valid partition has a state and a software_release.
    Scenario:
    1. Simulate computer containing one "destroyed" partition but with valid SR
    2. See if it destroyed
1605
    """
1606
    computer = ComputerForTest(self.software_root, self.instance_root)
1607 1608
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
1609

1610 1611 1612 1613
      computer.sequence = []
      instance.requested_state = 'destroyed'
      self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
      # Assert partition directory is empty
1614
      self.assertInstanceDirectoryListEqual(['0'])
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
      self.assertItemsEqual(os.listdir(instance.partition_path), [])
      self.assertItemsEqual(os.listdir(self.software_root),
                            [instance.software.software_hash])
      # Assert supervisor stopped process
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertIsNotCreated(wrapper_log)

      self.assertEqual(
          computer.sequence,
          ['/getFullComputerInformation', '/stoppedComputerPartition', '/destroyedComputerPartition'])
1625 1626 1627 1628 1629

  def test_slapgrid_not_destroy_bad_instance(self):
    """
    Checks that slapgrid-ur don't destroy instance not to be destroyed.
    """
Marco Mariani's avatar
Marco Mariani committed
1630
    computer = ComputerForTest(self.software_root, self.instance_root)
1631 1632 1633 1634 1635
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      instance.software.setBuildout(WRAPPER_CONTENT)
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
1636
      self.assertInstanceDirectoryListEqual(['0'])
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                             'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation',
                        '/availableComputerPartition',
                        '/startedComputerPartition'])
      self.assertEqual('started', instance.state)

      # Then run usage report and see if it is still working
      computer.sequence = []
      self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
1652 1653 1654
      # registerComputerPartition will create one more file: 
      from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
      request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
1655
      self.assertInstanceDirectoryListEqual(['0'])
1656 1657
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
1658 1659
                             'etc', 'software_release', 'worked',
                             '.slapos-retention-lock-delay', request_list_file])
1660 1661
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
1662
      self.assertInstanceDirectoryListEqual(['0'])
1663 1664
      self.assertItemsEqual(os.listdir(instance.partition_path),
                            ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
1665 1666
                             'etc', 'software_release', 'worked',
                             '.slapos-retention-lock-delay', request_list_file])
1667 1668 1669 1670 1671
      wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
      self.assertLogContent(wrapper_log, 'Working')
      self.assertEqual(computer.sequence,
                       ['/getFullComputerInformation'])
      self.assertEqual('started', instance.state)
1672

1673
  def test_slapgrid_instance_ignore_free_instance(self):
1674
    """
1675 1676
    Test than a free instance (so in "destroyed" state, but empty, without
    software_release URI) is ignored by slapgrid-cp.
1677 1678
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
1679 1680 1681
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.software.name = None
1682

1683 1684 1685 1686
      computer.sequence = []
      instance.requested_state = 'destroyed'
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      # Assert partition directory is empty
1687
      self.assertInstanceDirectoryListEqual(['0'])
1688 1689 1690
      self.assertItemsEqual(os.listdir(instance.partition_path), [])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
1691

1692 1693 1694 1695 1696 1697
  def test_slapgrid_report_ignore_free_instance(self):
    """
    Test than a free instance (so in "destroyed" state, but empty, without
    software_release URI) is ignored by slapgrid-ur.
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
1698 1699 1700
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.software.name = None
1701

1702 1703 1704 1705
      computer.sequence = []
      instance.requested_state = 'destroyed'
      self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
      # Assert partition directory is empty
1706
      self.assertInstanceDirectoryListEqual(['0'])
1707 1708 1709
      self.assertItemsEqual(os.listdir(instance.partition_path), [])
      self.assertItemsEqual(os.listdir(self.software_root), [instance.software.software_hash])
      self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
1710 1711


1712
class TestSlapgridSoftwareRelease(MasterMixin, unittest.TestCase):
1713 1714 1715 1716 1717 1718
  def test_one_software_buildout_fail_is_correctly_logged(self):
    """
    1. We set up a software using a corrupted buildout
    2. It will fail, make sure that whole log is sent to master
    """
    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
1719 1720
    with httmock.HTTMock(computer.request_handler):
      software = computer.software_list[0]
1721

1722 1723 1724
      line1 = "Nerdy kitten: Can I haz a process crash?"
      line2 = "Cedric: Sure, here it is."
      software.setBuildout("""#!/bin/sh
1725
echo %s; echo %s; exit 42""" % (line1, line2))
1726 1727 1728 1729 1730 1731 1732
      self.launchSlapgridSoftware()
      self.assertEqual(software.sequence,
                       ['/buildingSoftwareRelease', '/softwareReleaseError'])
      # We don't care of actual formatting, we just want to have full log
      self.assertIn(line1, software.error_log)
      self.assertIn(line2, software.error_log)
      self.assertIn('Failed to run buildout', software.error_log)
1733

1734

1735
class SlapgridInitialization(unittest.TestCase):
1736
  """
1737 1738
  "Abstract" class setting setup and teardown for TestSlapgridArgumentTuple
  and TestSlapgridConfigurationFile.
1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
  """

  def setUp(self):
    """
      Create the minimun default argument and configuration.
    """
    self.certificate_repository_path = tempfile.mkdtemp()
    self.fake_file_descriptor = tempfile.NamedTemporaryFile()
    self.slapos_config_descriptor = tempfile.NamedTemporaryFile()
    self.slapos_config_descriptor.write("""
[slapos]
software_root = /opt/slapgrid
instance_root = /srv/slapgrid
master_url = https://slap.vifib.com/
computer_id = your computer id
buildout = /path/to/buildout/binary
Marco Mariani's avatar
Marco Mariani committed
1755
""")
1756 1757 1758 1759 1760 1761
    self.slapos_config_descriptor.seek(0)
    self.default_arg_tuple = (
        '--cert_file', self.fake_file_descriptor.name,
        '--key_file', self.fake_file_descriptor.name,
        '--master_ca_file', self.fake_file_descriptor.name,
        '--certificate_repository_path', self.certificate_repository_path,
1762
        '-c', self.slapos_config_descriptor.name, '--now')
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775

    self.signature_key_file_descriptor = tempfile.NamedTemporaryFile()
    self.signature_key_file_descriptor.seek(0)

  def tearDown(self):
    """
      Removing the temp file.
    """
    self.fake_file_descriptor.close()
    self.slapos_config_descriptor.close()
    self.signature_key_file_descriptor.close()
    shutil.rmtree(self.certificate_repository_path, True)

1776

1777
class TestSlapgridCPWithMasterPromise(MasterMixin, unittest.TestCase):
1778
  def test_one_failing_promise(self):
Marco Mariani's avatar
Marco Mariani committed
1779
    computer = ComputerForTest(self.software_root, self.instance_root)
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      worked_file = os.path.join(instance.partition_path, 'fail_worked')
      fail = textwrap.dedent("""\
              #!/usr/bin/env sh
              touch "%s"
              exit 127""" % worked_file)
      instance.setPromise('fail', fail)
      self.assertEqual(self.grid.processComputerPartitionList(),
                       slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
      self.assertTrue(os.path.isfile(worked_file))
      self.assertTrue(instance.error)
      self.assertNotEqual('started', instance.state)
1794 1795

  def test_one_succeeding_promise(self):
1796
    computer = ComputerForTest(self.software_root, self.instance_root)
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      self.fake_waiting_time = 0.1
      worked_file = os.path.join(instance.partition_path, 'succeed_worked')
      succeed = textwrap.dedent("""\
              #!/usr/bin/env sh
              touch "%s"
              exit 0""" % worked_file)
      instance.setPromise('succeed', succeed)
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.isfile(worked_file))

      self.assertFalse(instance.error)
      self.assertEqual(instance.state, 'started')
1812 1813

  def test_stderr_has_been_sent(self):
Marco Mariani's avatar
Marco Mariani committed
1814
    computer = ComputerForTest(self.software_root, self.instance_root)
1815 1816
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
1817

1818 1819
      instance.requested_state = 'started'
      self.fake_waiting_time = 0.5
1820

1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834
      promise_path = os.path.join(instance.partition_path, 'etc', 'promise')
      os.makedirs(promise_path)
      succeed = os.path.join(promise_path, 'stderr_writer')
      worked_file = os.path.join(instance.partition_path, 'stderr_worked')
      with open(succeed, 'w') as f:
        f.write(textwrap.dedent("""\
              #!/usr/bin/env sh
              touch "%s"
              echo Error 1>&2
              exit 127""" % worked_file))
      os.chmod(succeed, 0o777)
      self.assertEqual(self.grid.processComputerPartitionList(),
                       slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
      self.assertTrue(os.path.isfile(worked_file))
1835

1836 1837 1838 1839 1840
      self.assertEqual(instance.error_log[-5:], 'Error')
      self.assertTrue(instance.error)
      self.assertIsNone(instance.state)

  def test_timeout_works(self):
Marco Mariani's avatar
Marco Mariani committed
1841
    computer = ComputerForTest(self.software_root, self.instance_root)
1842 1843
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
1844

1845 1846
      instance.requested_state = 'started'
      self.fake_waiting_time = 0.1
1847

1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
      promise_path = os.path.join(instance.partition_path, 'etc', 'promise')
      os.makedirs(promise_path)
      succeed = os.path.join(promise_path, 'timed_out_promise')
      worked_file = os.path.join(instance.partition_path, 'timed_out_worked')
      with open(succeed, 'w') as f:
        f.write(textwrap.dedent("""\
              #!/usr/bin/env sh
              touch "%s"
              sleep 5
              exit 0""" % worked_file))
      os.chmod(succeed, 0o777)
      self.assertEqual(self.grid.processComputerPartitionList(),
                       slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
1861
      self.assertTrue(os.path.isfile(worked_file))
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887

      self.assertTrue(instance.error)
      self.assertIsNone(instance.state)

  def test_two_succeeding_promises(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'

      self.fake_waiting_time = 0.1

      for i in range(2):
        worked_file = os.path.join(instance.partition_path, 'succeed_%s_worked' % i)
        succeed = textwrap.dedent("""\
              #!/usr/bin/env sh
              touch "%s"
              exit 0""" % worked_file)
        instance.setPromise('succeed_%s' % i, succeed)

      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      for i in range(2):
        worked_file = os.path.join(instance.partition_path, 'succeed_%s_worked' % i)
        self.assertTrue(os.path.isfile(worked_file))
      self.assertFalse(instance.error)
      self.assertEqual(instance.state, 'started')
1888

1889
  def test_one_succeeding_one_failing_promises(self):
Marco Mariani's avatar
Marco Mariani committed
1890
    computer = ComputerForTest(self.software_root, self.instance_root)
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      self.fake_waiting_time = 0.1

      for i in range(2):
        worked_file = os.path.join(instance.partition_path, 'promise_worked_%d' % i)
        lockfile = os.path.join(instance.partition_path, 'lock')
        promise = textwrap.dedent("""\
                  #!/usr/bin/env sh
                  touch "%(worked_file)s"
                  if [ ! -f %(lockfile)s ]
                  then
                    touch "%(lockfile)s"
                    exit 0
                  else
                    exit 127
                  fi""" % {
            'worked_file': worked_file,
            'lockfile': lockfile
        })
        instance.setPromise('promise_%s' % i, promise)
      self.assertEqual(self.grid.processComputerPartitionList(),
                       slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
      self.assertEquals(instance.error, 1)
      self.assertNotEqual('started', instance.state)
1917 1918

  def test_one_succeeding_one_timing_out_promises(self):
Marco Mariani's avatar
Marco Mariani committed
1919
    computer = ComputerForTest(self.software_root, self.instance_root)
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      self.fake_waiting_time = 0.1
      for i in range(2):
        worked_file = os.path.join(instance.partition_path, 'promise_worked_%d' % i)
        lockfile = os.path.join(instance.partition_path, 'lock')
        promise = textwrap.dedent("""\
                  #!/usr/bin/env sh
                  touch "%(worked_file)s"
                  if [ ! -f %(lockfile)s ]
                  then
                    touch "%(lockfile)s"
                  else
                    sleep 5
                  fi
                  exit 0""" % {
            'worked_file': worked_file,
            'lockfile': lockfile}
        )
        instance.setPromise('promise_%d' % i, promise)

      self.assertEqual(self.grid.processComputerPartitionList(),
                       slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)

      self.assertEquals(instance.error, 1)
      self.assertNotEqual(instance.state, 'started')
1947 1948 1949 1950 1951 1952 1953 1954

class TestSlapgridDestructionLock(MasterMixin, unittest.TestCase):
  def test_retention_lock(self):
    """
    Higher level test about actual retention (or no-retention) of instance
    if specifying a retention lock delay.
    """
    computer = ComputerForTest(self.software_root, self.instance_root)
1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.requested_state = 'started'
      instance.filter_dict = {'retention_delay': 1.0 / (3600 * 24)}
      self.grid.processComputerPartitionList()
      dummy_instance_file_path = os.path.join(instance.partition_path, 'dummy')
      with open(dummy_instance_file_path, 'w') as dummy_instance_file:
        dummy_instance_file.write('dummy')

      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.retention_lock_delay_filename
      )))

      instance.requested_state = 'destroyed'
      self.grid.agregateAndSendUsage()
      self.assertTrue(os.path.exists(dummy_instance_file_path))
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.retention_lock_date_filename
      )))

      self.grid.agregateAndSendUsage()
      self.assertTrue(os.path.exists(dummy_instance_file_path))

      time.sleep(1)
      self.grid.agregateAndSendUsage()
      self.assertFalse(os.path.exists(dummy_instance_file_path))

1984 1985 1986 1987 1988 1989

class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase):
  
  def setFirewallConfig(self, source_ip=""):
    firewall_conf= dict(
      authorized_sources=source_ip,
1990
      firewall_cmd='/bin/echo "no" #',
Alain Takoudjou's avatar
Alain Takoudjou committed
1991 1992
      firewall_executable='/bin/echo "service firewall started"',
      reload_config_cmd='/bin/echo "Config reloaded."',
1993 1994 1995 1996 1997
      log_file='fw-log.log',
      testing=True,
    )
    self.grid.firewall_conf = firewall_conf
  
1998 1999 2000
  def checkRuleFromIpSource(self, ip, accept_ip_list, cmd_list):
    # XXX - rules for one ip contain 2*len(ip_address_list + accept_ip_list) rules ACCEPT and 4 rules REJECT
    num_rules = len(self.ip_address_list) * 2 + len(accept_ip_list) * 2 + 4
2001
    self.assertEqual(len(cmd_list), num_rules)
2002
    base_cmd = '--permanent --direct --add-rule ipv4 filter'
2003

2004 2005
    # Check that there is REJECT rule on INPUT
    rule = '%s INPUT 1000 -d %s -j REJECT' % (base_cmd, ip)
2006 2007
    self.assertIn(rule, cmd_list)

2008 2009
    # Check that there is REJECT rule on FORWARD
    rule = '%s FORWARD 1000 -d %s -j REJECT' % (base_cmd, ip)
2010 2011
    self.assertIn(rule, cmd_list)

2012 2013
    # Check that there is REJECT rule on INPUT, ESTABLISHED,RELATED
    rule = '%s INPUT 900 -d %s -m state --state ESTABLISHED,RELATED -j REJECT' % (base_cmd, ip)
2014 2015
    self.assertIn(rule, cmd_list)

2016 2017
    # Check that there is REJECT rule on FORWARD, ESTABLISHED,RELATED
    rule = '%s FORWARD 900 -d %s -m state --state ESTABLISHED,RELATED -j REJECT' % (base_cmd, ip)
2018 2019 2020 2021 2022 2023
    self.assertIn(rule, cmd_list)
    
    # Check that there is INPUT ACCEPT on ip_list
    for _, other_ip in self.ip_address_list:
      rule = '%s INPUT 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
2024 2025
      rule = '%s FORWARD 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
2026 2027

    # Check that there is FORWARD ACCEPT on ip_list
2028 2029 2030
    for other_ip in accept_ip_list:
      rule = '%s INPUT 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
2031 2032
      rule = '%s FORWARD 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
2033 2034 2035
  
  def checkRuleFromIpSourceReject(self, ip, reject_ip_list, cmd_list):
    # XXX - rules for one ip contain 2 + 2*len(ip_address_list) rules ACCEPT and 4*len(reject_ip_list) rules REJECT
2036
    num_rules = (len(self.ip_address_list) * 2) + (len(reject_ip_list) * 4)
2037
    self.assertEqual(len(cmd_list), num_rules)
2038
    base_cmd = '--permanent --direct --add-rule ipv4 filter'
2039 2040

    # Check that there is ACCEPT rule on INPUT
2041 2042
    #rule = '%s INPUT 0 -d %s -j ACCEPT' % (base_cmd, ip)
    #self.assertIn(rule, cmd_list)
2043 2044

    # Check that there is ACCEPT rule on FORWARD
2045 2046
    #rule = '%s FORWARD 0 -d %s -j ACCEPT' % (base_cmd, ip)
    #self.assertIn(rule, cmd_list)
2047 2048 2049
    
    # Check that there is INPUT/FORWARD ACCEPT on ip_list
    for _, other_ip in self.ip_address_list:
2050
      rule = '%s INPUT 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
2051
      self.assertIn(rule, cmd_list)
2052
      rule = '%s FORWARD 0 -s %s -d %s -j ACCEPT' % (base_cmd, other_ip, ip)
2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
      self.assertIn(rule, cmd_list)

    # Check that there is INPUT/FORWARD REJECT on ip_list
    for other_ip in reject_ip_list:
      rule = '%s INPUT 900 -s %s -d %s -j REJECT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
      rule = '%s FORWARD 900 -s %s -d %s -j REJECT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
      rule = '%s INPUT 800 -s %s -d %s -m state --state ESTABLISHED,RELATED -j REJECT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
      rule = '%s FORWARD 800 -s %s -d %s -m state --state ESTABLISHED,RELATED -j REJECT' % (base_cmd, other_ip, ip)
      self.assertIn(rule, cmd_list)
2065 2066 2067 2068 2069 2070

  def test_getFirewallRules(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    self.ip_address_list = computer.ip_address_list
    ip = computer.instance_list[0].full_ip_list[0][1]
2071 2072 2073 2074 2075 2076 2077
    source_ip_list = ['10.32.0.15', '10.32.0.0/8']
    
    cmd_list = self.grid._getFirewallAcceptRules(ip,
                                [elt[1] for elt in self.ip_address_list],
                                source_ip_list,
                                ip_type='ipv4')
    self.checkRuleFromIpSource(ip, source_ip_list, cmd_list)
2078
    
2079
    cmd_list = self.grid._getFirewallRejectRules(ip,
2080
                                [elt[1] for elt in self.ip_address_list],
2081
                                source_ip_list,
2082
                                ip_type='ipv4')
2083
    self.checkRuleFromIpSourceReject(ip, source_ip_list, cmd_list)
2084 2085 2086 2087 2088


  def test_checkAddFirewallRules(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
2089
    # For simulate query rule success
2090
    self.grid.firewall_conf['firewall_cmd'] = '/bin/echo "no" #'
2091 2092 2093 2094 2095
    self.ip_address_list = computer.ip_address_list
    instance = computer.instance_list[0]
    ip = instance.full_ip_list[0][1]
    name = computer.instance_list[0].name
    
2096
    cmd_list = self.grid._getFirewallAcceptRules(ip,
2097
                                [elt[1] for elt in self.ip_address_list],
2098
                                [],
2099 2100 2101
                                ip_type='ipv4')
    self.grid._checkAddFirewallRules(name, cmd_list, add=True)

2102 2103 2104 2105
    rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
2106 2107
    with open(rules_path, 'r') as frules:
      rules_list = json.loads(frules.read())
2108
      self.checkRuleFromIpSource(ip, [], rules_list)
2109 2110

    # Remove all rules
2111
    self.grid.firewall_conf['firewall_cmd'] = '/bin/echo "yes" #'
2112 2113 2114 2115 2116 2117
    self.grid._checkAddFirewallRules(name, cmd_list, add=False)
    with open(rules_path, 'r') as frules:
      rules_list = json.loads(frules.read())
      self.assertEqual(rules_list, [])

    # Add one more ip in the authorized list
2118
    self.grid.firewall_conf['firewall_cmd'] = '/bin/echo "no" #'
2119
    self.ip_address_list.append(('interface1', '10.0.8.7'))
2120
    cmd_list = self.grid._getFirewallAcceptRules(ip,
2121
                                [elt[1] for elt in self.ip_address_list],
2122
                                [],
2123 2124 2125 2126 2127
                                ip_type='ipv4')

    self.grid._checkAddFirewallRules(name, cmd_list, add=True)
    with open(rules_path, 'r') as frules:
      rules_list = json.loads(frules.read())
2128
      self.checkRuleFromIpSource(ip, [], rules_list)
2129 2130 2131 2132 2133 2134 2135 2136 2137

  def test_partition_no_firewall(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      self.assertEqual(self.grid.processComputerPartitionList(),
                        slapgrid.SLAPGRID_SUCCESS)
      self.assertFalse(os.path.exists(os.path.join(
          instance.partition_path,
2138
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2139 2140
      )))

2141 2142 2143 2144 2145 2146 2147 2148
  def test_partition_firewall_restrict(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2149
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161
      )))
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
      self.ip_address_list = computer.ip_address_list
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())
      
      ip = instance.full_ip_list[0][1]
      self.checkRuleFromIpSource(ip, [], rules_list)

2162 2163 2164 2165 2166
  def test_partition_firewall(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
2167
      instance.filter_dict = {'fw_restricted_access': 'off'}
2168 2169 2170
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2171
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2172
      )))
2173 2174 2175 2176
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
2177 2178 2179 2180 2181
      self.ip_address_list = computer.ip_address_list
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())
      
      ip = instance.full_ip_list[0][1]
2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194
      self.checkRuleFromIpSourceReject(ip, [], rules_list)

  @unittest.skip('Always fail: instance.filter_dict can\'t change')
  def test_partition_firewall_restricted_access_change(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.filter_dict = {'fw_restricted_access': 'off',
                              'fw_rejected_sources': '10.0.8.11'}
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2195
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
      )))
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
      self.ip_address_list = computer.ip_address_list
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())

      ip = instance.full_ip_list[0][1]
      self.checkRuleFromIpSourceReject(ip, ['10.0.8.11'], rules_list)

2208 2209
      # For remove rules
      self.grid.firewall_conf['firewall_cmd'] = '/bin/echo "yes" #'
2210 2211 2212 2213 2214 2215 2216 2217 2218 2219
      instance.setFilterParameter({'fw_restricted_access': 'on',
                              'fw_authorized_sources': ''})
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)

      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())

      self.checkRuleFromIpSource(ip, [], rules_list)

  def test_partition_firewall_ipsource_accept(self):
2220 2221
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
2222 2223
    source_ip = ['10.0.8.10', '10.0.8.11']
    self.grid.firewall_conf['authorized_sources'] = [source_ip[0]]
2224 2225
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
2226 2227
      instance.filter_dict = {'fw_restricted_access': 'on',
                              'fw_authorized_sources': source_ip[1]}
2228 2229 2230
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2231
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2232
      )))
2233 2234 2235 2236
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
2237
      rules_list= []
2238
      self.ip_address_list = computer.ip_address_list
2239
      ip = instance.full_ip_list[0][1]
2240
      base_cmd = '--permanent --direct --add-rule ipv4 filter'
2241 2242 2243
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())

2244 2245 2246 2247 2248 2249
      for thier_ip in source_ip:
        rule_input = '%s INPUT 0 -s %s -d %s -j ACCEPT' % (base_cmd, thier_ip, ip)
        self.assertIn(rule_input, rules_list)
  
        rule_fwd = '%s FORWARD 0 -s %s -d %s -j ACCEPT' % (base_cmd, thier_ip, ip)
        self.assertIn(rule_fwd, rules_list)
2250

2251
      self.checkRuleFromIpSource(ip, source_ip, rules_list)
2252

2253
  def test_partition_firewall_ipsource_reject(self):
2254 2255 2256
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    source_ip = '10.0.8.10'
2257 2258
    
    self.grid.firewall_conf['authorized_sources'] = ['10.0.8.15']
2259 2260
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
2261 2262
      instance.filter_dict = {'fw_rejected_sources': source_ip,
                              'fw_restricted_access': 'off'}
2263 2264 2265
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2266
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2267
      )))
2268 2269 2270 2271
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
2272
      rules_list= []
2273 2274
      self.ip_address_list = computer.ip_address_list
      self.ip_address_list.append(('iface', '10.0.8.15'))
2275
      ip = instance.full_ip_list[0][1]
2276
      base_cmd = '--permanent --direct --add-rule ipv4 filter'
2277 2278 2279
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())

2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293
      self.checkRuleFromIpSourceReject(ip, source_ip.split(' '), rules_list)

  def test_partition_firewall_ip_change(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    self.setFirewallConfig()
    source_ip = ['10.0.8.10', '10.0.8.11']
    self.grid.firewall_conf['authorized_sources'] = [source_ip[0]]
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      instance.filter_dict = {'fw_restricted_access': 'on',
                              'fw_authorized_sources': source_ip[1]}
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertTrue(os.path.exists(os.path.join(
          instance.partition_path,
2294
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
      )))
      rules_path = os.path.join(
          instance.partition_path,
          slapos.grid.SlapObject.Partition.partition_firewall_rules_name
      )
      rules_list= []
      self.ip_address_list = computer.ip_address_list
      ip = instance.full_ip_list[0][1]
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())

      self.checkRuleFromIpSource(ip, source_ip, rules_list)
      instance = computer.instance_list[0]
      # XXX -- removed
      #instance.filter_dict = {'fw_restricted_access': 'on',
      #                        'fw_authorized_sources': source_ip[0]}
2311

2312 2313
      # For simulate query rule exist
      self.grid.firewall_conf['firewall_cmd'] = '/bin/echo "yes" #'
2314 2315 2316 2317
      self.grid.firewall_conf['authorized_sources'] = []
      computer.ip_address_list.append(('route_interface1', '10.10.8.4'))
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.ip_address_list = computer.ip_address_list
2318

2319 2320 2321
      with open(rules_path, 'r') as frules:
        rules_list = json.loads(frules.read())
      self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list)
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339

class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase):

from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME

  def test_one_partition(self):
    computer = ComputerForTest(self.software_root, self.instance_root)
    with httmock.HTTMock(computer.request_handler):
      instance = computer.instance_list[0]
      partition = os.path.join(self.instance_root, '0')
      request_list_file = os.path.join(partition,
          COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
      with open(request_list_file, 'w') as f:
        f.write('some partition')
      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
      self.assertInstanceDirectoryListEqual(['0'])

      self.assertFalse(os.path.exists(request_list_file)