format.py 39.3 KB
Newer Older
Łukasz Nowak's avatar
Łukasz Nowak committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010, 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
from optparse import OptionParser, Option
from xml_marshaller import xml_marshaller
30
from pwd import getpwnam
Łukasz Nowak's avatar
Łukasz Nowak committed
31 32 33 34 35 36 37 38 39 40 41 42 43
import ConfigParser
import grp
import logging
import netaddr
import netifaces
import os
import random
import slapos.slap as slap
import socket
import subprocess
import sys
import time

Vincent Pelletier's avatar
Vincent Pelletier committed
44 45 46 47
class OS(object):
  _os = os

  def __init__(self, config):
48
    self._dry_run = config.dry_run
Vincent Pelletier's avatar
Vincent Pelletier committed
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
    self._verbose = config.verbose
    self._logger = config.logger
    add = self._addWrapper
    add('chown')
    add('chmod')
    add('makedirs')
    add('mkdir')

  def _addWrapper(self, name):
    def wrapper(*args, **kw):
      if self._verbose:
        arg_list = [repr(x) for x in args] + [
          '%s=%r' % (x, y) for x, y in kw.iteritems()]
        self._logger.debug('%s(%s)' % (
          name,
          ', '.join(arg_list)
        ))
66 67
      if not self._dry_run:
        getattr(self._os, name)(*args, **kw)
Vincent Pelletier's avatar
Vincent Pelletier committed
68 69 70 71
    setattr(self, name, wrapper)

  def __getattr__(self, name):
    return getattr(self._os, name)
Łukasz Nowak's avatar
Łukasz Nowak committed
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172

class SlapError(Exception):
  """
  Slap error
  """
  def __init__(self, message):
    self.msg = message

class UsageError(SlapError):
  pass

class ExecError(SlapError):
  pass

def callAndRead(argument_list, raise_on_error=True):
  popen = subprocess.Popen(argument_list, stdout=subprocess.PIPE,
      stderr=subprocess.STDOUT)
  result = popen.communicate()[0]
  if raise_on_error and popen.returncode != 0:
    raise ValueError('Issue during invoking %r, result was:\n%s' % (argument_list, result))
  return popen.returncode, result

def isGlobalScopeAddress(a):
  """Returns True if a is global scope IP v4/6 address"""
  ip = netaddr.IPAddress(a)
  return not ip.is_link_local() and not ip.is_loopback() and \
      not ip.is_reserved() and ip.is_unicast()

def netmaskToPrefixIPv4(netmask):
  """Convert string represented netmask to its integer prefix"""
  return netaddr.strategy.ipv4.netmask_to_prefix[
          netaddr.strategy.ipv4.str_to_int(netmask)]

def netmaskToPrefixIPv6(netmask):
  """Convert string represented netmask to its integer prefix"""
  return netaddr.strategy.ipv6.netmask_to_prefix[
          netaddr.strategy.ipv6.str_to_int(netmask)]

def _getDict(instance):
  """
  Serialize an object instance into dictionaries. List and dict will remains
  the same, basic type too. But encapsulated object will be returned as dict.
  Set, collections and other aren't handle for now.

  Args:
    instance: an object of any type.

  Returns:
    A dictionary if the given object wasn't a list, a list otherwise.
  """
  if isinstance(instance, list):
    return [_getDict(item) for item in instance]

  elif isinstance(instance, dict):
    result = {}
    for key in instance.keys():
      result[key] = _getDict(instance[key])
    return result

  else:
    try:

      result = {}
      for key in instance.__dict__.keys():
        result[key] = _getDict(instance.__dict__[key])
      return result

    except AttributeError:
      return instance

class Error(Exception):
  "Base class for exceptions in this module."
  def __str__(self):
    return self.message

class NoAddressOnBridge(Error):
  """
  Exception raised if there's not address on the bridge to construct IPv6
  address with.

  Attributes:
    brige: String, the name of the bridge.
  """

  def __init__(self, bridge):
    self.message = 'No IPv6 found on bridge %s to construct IPv6 with.' % bridge

class AddressGenerationError(Error):
  """
  Exception raised if the generation of an IPv6 based on the prefix obtained
  from the bridge failed.

  Attributes:
    addr: String, the invalid address the exception is raised for.
  """
  def __init__(self, addr):
    self.message = 'Generated IPv6 %s seems not to be a valid IP.' % addr

class Computer:
  "Object representing the computer"

Łukasz Nowak's avatar
Łukasz Nowak committed
173 174
  def __init__(self, reference, bridge=None, addr = None, netmask = None,
    ipv6_interface=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
175 176 177 178 179 180 181 182 183 184
    """
    Attributes:
      reference: String, the reference of the computer.
      bridge: String, if it has one, the name of the computer's bridge.
    """
    self.reference = str(reference)
    self.bridge = bridge
    self.partition_list = []
    self.address = addr
    self.netmask = netmask
Łukasz Nowak's avatar
Łukasz Nowak committed
185
    self.ipv6_interface = ipv6_interface
Łukasz Nowak's avatar
Łukasz Nowak committed
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211

  def __getinitargs__(self):
    return (self.reference, self.bridge)

  def getAddress(self):
    """
    Return a list of the bridge address not attributed to any partition, (which
    are therefore free for the computer itself).

    Returns:
      False if the bridge isn't available, else the list of the free addresses.
    """
    if self.bridge is None:
      return dict(addr=self.address, netmask=self.netmask)

    computer_partition_address_list = []
    for partition in self.partition_list:
      for address in partition.address_list:
        if netaddr.valid_ipv6(address['addr']):
          computer_partition_address_list.append(address['addr'])
    # Going through addresses of the computer's bridge interface
    for address_dict in self.bridge.getGlobalScopeAddressList():
      # Comparing with computer's partition addresses
      if address_dict['addr'] not in computer_partition_address_list:
        return address_dict

212 213
    # all addresses on interface are for partition, so lets add new one
    computer_tap = Tap('compdummy')
Łukasz Nowak's avatar
Łukasz Nowak committed
214
    computer_tap.createWithOwner(User('root'), attach_to_tap=True)
215 216
    self.bridge.addTap(computer_tap)
    return self.bridge.addAddr()
Łukasz Nowak's avatar
Łukasz Nowak committed
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231

  def send(self, config):
    """
    Send a marshalled dictionary of the computer object serialized via_getDict.
    """

    slap_instance = slap.slap()
    connection_dict = {}
    if config.key_file and config.cert_file:
      connection_dict.update(
        key_file=config.key_file,
        cert_file=config.cert_file)
    slap_instance.initializeConnection(config.master_url,
      **connection_dict)
    slap_computer = slap_instance.registerComputer(self.reference)
232 233
    if config.dry_run:
      return
Łukasz Nowak's avatar
Łukasz Nowak committed
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
    return slap_computer.updateConfiguration(
        xml_marshaller.dumps(_getDict(self)))

  def dump(self, path_to_xml):
    """
    Dump the computer object to an xml file via xml_marshaller.

    Args:
      path_to_xml: String, path to the file to load.
      users: List of User, list of user needed to be add to the dump
          (even if they are not related to any tap interface).
    """

    computer_dict = _getDict(self)
    output_file = open(path_to_xml,'w')
    output_file.write(xml_marshaller.dumps(computer_dict))
    output_file.close()

  @classmethod
Łukasz Nowak's avatar
Łukasz Nowak committed
253
  def load(cls, path_to_xml, reference, ipv6_interface):
Łukasz Nowak's avatar
Łukasz Nowak committed
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
    """
    Create a computer object from a valid xml file.

    Arg:
      path_to_xml: String, a path to a valid file containing
          a valid configuration.

    Return:
      A Computer object if the path where pointing on a valid
          file, False otherwise.
    """

    dumped_dict = xml_marshaller.loads(open(path_to_xml).read())

    # Reconstructing the computer object from the xml
    computer = Computer(
        reference = reference,
        addr = dumped_dict['address'],
        netmask = dumped_dict['netmask'],
Łukasz Nowak's avatar
Łukasz Nowak committed
273
        ipv6_interface=ipv6_interface,
Łukasz Nowak's avatar
Łukasz Nowak committed
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
    )

    for partition_dict in dumped_dict['partition_list']:

      if partition_dict['user']:
        user = User(partition_dict['user']['name'])
      else:
        user = User('root')

      if partition_dict['tap']:
        tap = Tap(partition_dict['tap']['name'])
      else:
        tap = Tap(partition_dict['reference'])

      address_list = partition_dict['address_list']

      partition = Partition(
          reference = partition_dict['reference'],
          path = partition_dict['path'],
          user = user,
          address_list = address_list,
          tap = tap,
      )

      computer.partition_list.append(partition)

    return computer

  def construct(self, alter_user=True, alter_network=True):
    """
    Construct the computer object as it is.
    """
    if alter_network and self.address is not None:
      self.bridge.addAddr(self.address, self.netmask)

    for path in self.instance_root, self.software_root: 
      if not os.path.exists(path):
        os.makedirs(path, 0755)
      else:
        os.chmod(path, 0755)

    # own self.software_root by slapsoft
    slapsoft = User('slapsoft')
    slapsoft.path = self.software_root
    if alter_user:
      slapsoft.create()
320
      slapsoft_pw = getpwnam(slapsoft.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
321 322 323
      os.chown(self.software_root, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
    os.chmod(self.software_root, 0755)

324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
    try:
      for partition_index, partition in enumerate(self.partition_list):
        # Reconstructing User's
        partition.path = os.path.join(self.instance_root, partition.reference)
        partition.user.setPath(partition.path)
        partition.user.additional_group_list = [slapsoft.name]
        if alter_user:
          partition.user.create()

        # Reconstructing Tap
        if partition.user and partition.user.isAvailable():
          owner = partition.user
        else:
          owner = User('root')

        if alter_network:
          # In case it has to be  attached to the TAP network device, only one
          # is necessary for the bridge to assert carrier
          if self.bridge.attach_to_tap and partition_index == 0:
            partition.tap.createWithOwner(owner, attach_to_tap=True)
Łukasz Nowak's avatar
Łukasz Nowak committed
344
          else:
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
            partition.tap.createWithOwner(owner)

          self.bridge.addTap(partition.tap)

        # Reconstructing partition's directory
        partition.createPath(alter_user)

        # Reconstructing partition's address
        # There should be two addresses on each Computer Partition:
        #  * global IPv6
        #  * local IPv4, took from slapformat:ipv4_local_network
        if len(partition.address_list) == 0:
          # regenerate
          partition.address_list.append(self.bridge.addIPv4LocalAddress())
          partition.address_list.append(self.bridge.addAddr())
        elif alter_network:
          # regenerate list of addresses
          old_partition_address_list = partition.address_list
          partition.address_list = []
          if len(old_partition_address_list) != 2:
            raise ValueError('There should be exactly 2 stored addresses')
          if not any([netaddr.valid_ipv6(q['addr']) for q in old_partition_address_list]):
            raise ValueError('Not valid ipv6 addresses loaded')
          if not any([netaddr.valid_ipv4(q['addr']) for q in old_partition_address_list]):
            raise ValueError('Not valid ipv6 addresses loaded')
          for address in old_partition_address_list:
            if netaddr.valid_ipv6(address['addr']):
              partition.address_list.append(self.bridge.addAddr(address['addr'],
                                                                address['netmask']))
            elif netaddr.valid_ipv4(address['addr']):
              partition.address_list.append(self.bridge.addIPv4LocalAddress(address['addr']))
            else:
              raise ValueError('Address %r is incorrect' % address['addr'])
    finally:
379
      if alter_network and self.bridge.attach_to_tap:
380 381 382 383
        try:
          self.partition_list[0].tap.detach()
        except IndexError:
          pass
Łukasz Nowak's avatar
Łukasz Nowak committed
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417

class Partition:
  "Represent a computer partition"

  def __init__(self, reference, path, user, address_list, tap):
    """
    Attributes:
      reference: String, the name of the partition.
      path: String, the path to the partition folder.
      user: User, the user linked to this partition.
      address_list: List of associated IP addresses.
      tap: Tap, the tap interface linked to this partition.
    """

    self.reference = str(reference)
    self.path = str(path)
    self.user = user
    self.address_list = address_list or []
    self.tap = tap

  def __getinitargs__(self):
    return (self.reference, self.path, self.user, self.address_list, self.tap)

  def createPath(self, alter_user=True):
    """
    Create the directory of the partition, assign to the partition user and give
    it the 750 permission. In case if path exists just modifies it.
    """

    self.path = os.path.abspath(self.path)
    owner = self.user if self.user else User('root')
    if not os.path.exists(self.path):
      os.mkdir(self.path, 0750)
    if alter_user:
418
      owner_pw = getpwnam(owner.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
      os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
    os.chmod(self.path, 0750)

class User:
  "User: represent and manipulate a user on the system."

  def __init__(self, user_name, additional_group_list=None):
    """
    Attributes:
        user_name: string, the name of the user, who will have is home in
    """
    self.name = str(user_name)
    self.additional_group_list = additional_group_list

  def __getinitargs__(self):
    return (self.name,)

  def setPath(self, path):
    self.path = path

  def create(self):
    """
    Create a user on the system who will be named after the self.name with its
    own group and directory.

    Returns:
        True: if the user creation went right
    """
    # XXX: This method shall be no-op in case if all is correctly setup
    #      This method shall check if all is correctly done
    #      This method shall not reset groups, just add them
    try:
      grp.getgrnam(self.name)
    except KeyError:
      callAndRead(['groupadd', self.name])

455
    user_parameter_list = ['-d', self.path, '-g', self.name, '-s', '/bin/false']
Łukasz Nowak's avatar
Łukasz Nowak committed
456 457 458 459
    if self.additional_group_list is not None:
      user_parameter_list.extend(['-G', ','.join(self.additional_group_list)])
    user_parameter_list.append(self.name)
    try:
460
      getpwnam(self.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
    except KeyError:
      callAndRead(['useradd'] + user_parameter_list)
    else:
      callAndRead(['usermod'] + user_parameter_list)

    return True

  def isAvailable(self):
    """
    Determine the availability of a user on the system

    Return:
        True: if available
        False: otherwise
    """

    try:
478
      getpwnam(self.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
479 480 481 482 483
      return True

    except KeyError:
      return False

484 485 486 487 488
import struct
import fcntl
import errno
import threading

Łukasz Nowak's avatar
Łukasz Nowak committed
489 490
class Tap:
  "Tap represent a tap interface on the system"
491 492 493
  IFF_TAP = 0x0002
  TUNSETIFF = 0x400454ca
  KEEP_TAP_ATTACHED_EVENT = threading.Event()
Łukasz Nowak's avatar
Łukasz Nowak committed
494 495 496 497 498 499 500 501 502 503 504 505 506

  def __init__(self, tap_name):
    """
    Attributes:
        tap_name: String, the name of the tap interface.
        user: User, the owner of the tap interface.
    """

    self.name = str(tap_name)

  def __getinitargs__(self):
    return (self.name,)

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
  def attach(self):
    """
    Attach to the TAP interface, meaning  that it just opens the TAP interface
    and wait for the caller to notify that it can be safely detached.

    Linux  distinguishes administrative  and operational  state of  an network
    interface.  The  former can be set  manually by running ``ip  link set dev
    <dev> up|down'', whereas the latter states that the interface can actually
    transmit  data (for  a wired  network interface,  it basically  means that
    there is  carrier, e.g.  the network  cable is plugged  into a  switch for
    example).

    In order  to be able to check  the uniqueness of IPv6  address assigned to
    the bridge, the network interface  must be up from an administrative *and*
    operational point of view.

    However,  from  Linux  2.6.39,  the  bridge  reflects  the  state  of  the
    underlying device (e.g.  the bridge asserts carrier if at least one of its
    ports has carrier) whereas it  always asserted carrier before. This should
    work fine for "real" network interface,  but will not work properly if the
    bridge only binds TAP interfaces, which, from 2.6.36, reports carrier only
    and only if an userspace program is attached.
    """
    tap_fd = os.open("/dev/net/tun", os.O_RDWR)

    try:
      # Attach to the TAP interface which has previously been created
      fcntl.ioctl(tap_fd, self.TUNSETIFF,
                  struct.pack("16sI", self.name, self.IFF_TAP))

    except IOError, error:
      # If  EBUSY, it  means another  program is  already attached,  thus just
      # ignore it...
      if error.errno != errno.EBUSY:
        os.close(tap_fd)
        raise
    else:
      # Block until the  caller send an event stating that  the program can be
      # now detached safely,  thus bringing down the TAP  device (from 2.6.36)
      # and the bridge at the same time (from 2.6.39)
      self.KEEP_TAP_ATTACHED_EVENT.wait()
    finally:
      os.close(tap_fd)

  def detach(self):
    """
    Detach to the  TAP network interface by notifying  the thread which attach
    to the TAP and closing the TAP file descriptor
    """
    self.KEEP_TAP_ATTACHED_EVENT.set()

  def createWithOwner(self, owner, attach_to_tap=False):
Łukasz Nowak's avatar
Łukasz Nowak committed
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
    """
    Create a tap interface on the system.

    Return:
        True: Everything went right.
    """

    # some systems does not have -p switch for tunctl
    #callAndRead(['tunctl', '-p', '-t', self.name, '-u', owner.name])
    check_file = '/sys/devices/virtual/net/%s/owner' % self.name
    owner_id = None
    if os.path.exists(check_file):
      try:
        owner_id = int(open(check_file).read().strip())
      except Exception:
        pass
575
    if (owner_id is None) or (owner_id != getpwnam(owner.name).pw_uid):
Łukasz Nowak's avatar
Łukasz Nowak committed
576 577 578
      callAndRead(['tunctl', '-t', self.name, '-u', owner.name])
    callAndRead(['ip', 'link', 'set', self.name, 'up'])

579 580 581
    if attach_to_tap:
      threading.Thread(target=self.attach).start()

Łukasz Nowak's avatar
Łukasz Nowak committed
582 583 584 585 586
    return True

class Bridge:
  "Bridge represent a bridge on the system"

Łukasz Nowak's avatar
Łukasz Nowak committed
587
  def __init__(self, name, ipv4_local_network, ipv6_interface):
Łukasz Nowak's avatar
Łukasz Nowak committed
588 589 590 591 592 593 594
    """
    Attributes:
        name: String, the name of the bridge
    """

    self.name = str(name)
    self.ipv4_local_network = ipv4_local_network
Łukasz Nowak's avatar
Łukasz Nowak committed
595
    self.ipv6_interface = ipv6_interface
Łukasz Nowak's avatar
Łukasz Nowak committed
596

597 598 599 600 601
    # Attach to TAP  network interface, only if the  bridge interface does not
    # report carrier
    returncode, result = callAndRead(['ip', 'addr', 'list', self.name])
    self.attach_to_tap = 'DOWN' in result.split('\n', 1)[0]

Łukasz Nowak's avatar
Łukasz Nowak committed
602 603 604 605 606 607 608 609 610 611 612 613 614 615
  def __getinitargs__(self):
    return (self.name,)

  def getIPv4LocalAddressList(self):
    """Returns currently configured local IPv4 addresses which are in ipv4_local_network"""
    if not socket.AF_INET in netifaces.ifaddresses(self.name):
      return []
    return [dict(addr=q['addr'], netmask=q['netmask']) for q in
      netifaces.ifaddresses(self.name)[socket.AF_INET] if netaddr.IPAddress(
        q['addr'], 4) in netaddr.glob_to_iprange(
          netaddr.cidr_to_glob(self.ipv4_local_network))]

  def getGlobalScopeAddressList(self):
    """Returns currently configured global scope IPv6 addresses"""
Łukasz Nowak's avatar
Łukasz Nowak committed
616 617 618 619
    if self.ipv6_interface:
      interface_name = self.ipv6_interface
    else:
      interface_name = self.name
620
    try:
Łukasz Nowak's avatar
Łukasz Nowak committed
621
      address_list = [q for q in netifaces.ifaddresses(interface_name)[socket.AF_INET6]
622 623 624
                      if isGlobalScopeAddress(q['addr'].split('%')[0])]
    except KeyError:
      raise ValueError("%s must have at least one IPv6 address assigned" % \
Łukasz Nowak's avatar
Łukasz Nowak committed
625
                         interface_name)
Łukasz Nowak's avatar
Łukasz Nowak committed
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
    # XXX: Missing implementation of Unique Local IPv6 Unicast Addresses as
    # defined in http://www.rfc-editor.org/rfc/rfc4193.txt
    # XXX: XXX: XXX: IT IS DISALLOWED TO IMPLEMENT link-local addresses as
    # Linux and BSD are possibly wrongly implementing it -- it is "too local"
    # it is impossible to listen or access it on same node
    # XXX: IT IS DISALLOWED to implement ad hoc solution like inventing node
    # local addresses or anything which does not exists in RFC!
    return address_list

  def getInterfaceList(self):
    """Returns list of interfaces already present on bridge"""
    interface_list = []
    returncode, result = callAndRead(['brctl', 'show'])
    in_bridge = False
    for line in result.split('\n'):
      if len(line.split()) > 1:
        if self.name in line:
          interface_list.append(line.split()[-1])
          in_bridge = True
          continue
        if in_bridge:
          break
      elif in_bridge:
        if line.strip():
          interface_list.append(line.strip())

    return interface_list

  def addTap(self, tap):
    """
    Add the tap interface tap to the bridge.

    Args:
      tap: Tap, the tap interface.
    """
    if tap.name not in self.getInterfaceList():
      callAndRead(['brctl', 'addif', self.name, tap.name])

  def _addSystemAddress(self, address, netmask, ipv6=True):
    """Adds system address to bridge
    
    Returns True if address was added successfully.

    Returns False if there was issue.
    """
    if ipv6:
      address_string = '%s/%s' % (address, netmaskToPrefixIPv6(netmask))
      af = socket.AF_INET6
Łukasz Nowak's avatar
Łukasz Nowak committed
674 675 676 677
      if self.ipv6_interface:
        interface_name = self.ipv6_interface
      else:
        interface_name = self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
678 679 680
    else:
      af = socket.AF_INET
      address_string = '%s/%s' % (address, netmaskToPrefixIPv4(netmask))
Łukasz Nowak's avatar
Łukasz Nowak committed
681
      interface_name = self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
682 683 684

    # check if address is already took by any other interface
    for interface in netifaces.interfaces():
Łukasz Nowak's avatar
Łukasz Nowak committed
685
      if interface != interface_name:
Łukasz Nowak's avatar
Łukasz Nowak committed
686 687 688 689 690
        address_dict = netifaces.ifaddresses(interface)
        if af in address_dict:
         if address in [q['addr'].split('%')[0] for q in address_dict[af]]:
            return False

Łukasz Nowak's avatar
Łukasz Nowak committed
691
    if not af in netifaces.ifaddresses(interface_name) or not address in [q['addr'].split('%')[0] for q in netifaces.ifaddresses(interface_name)[af]]:
Łukasz Nowak's avatar
Łukasz Nowak committed
692
      # add an address
Łukasz Nowak's avatar
Łukasz Nowak committed
693
      callAndRead(['ip', 'addr', 'add', address_string, 'dev', interface_name])
Łukasz Nowak's avatar
Łukasz Nowak committed
694 695 696
      # wait few moments
      time.sleep(2)
    # check existence on interface
Łukasz Nowak's avatar
Łukasz Nowak committed
697
    returncode, result = callAndRead(['ip', 'addr', 'list', interface_name])
Łukasz Nowak's avatar
Łukasz Nowak committed
698 699 700 701
    for l in result.split('\n'):
      if address in l:
        if 'tentative' in l:
          # duplicate, remove
Łukasz Nowak's avatar
Łukasz Nowak committed
702 703
          callAndRead(['ip', 'addr', 'del', address_string, 'dev',
            interface_name])
Łukasz Nowak's avatar
Łukasz Nowak committed
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762
          return False
        # found and clean
        return True
    # even when added not found, this is bad...
    return False

  def _generateRandomIPv4Address(self, netmask):
    # no addresses found, generate new one
    # Try 10 times to add address, raise in case if not possible
    try_num = 10
    while try_num > 0:
      addr = random.choice([q for q in netaddr.glob_to_iprange(
        netaddr.cidr_to_glob(self.ipv4_local_network))]).format()
      if dict(addr=addr, netmask=netmask) not in self.getIPv4LocalAddressList():
        # Checking the validity of the IPv6 address
        if self._addSystemAddress(addr, netmask, False):
          return dict(addr=addr, netmask=netmask)
        try_num -= 1

    raise AddressGenerationError(addr)

  def addIPv4LocalAddress(self, addr=None):
    """Adds local IPv4 address in ipv4_local_network"""
    netmask = '255.255.255.255'
    local_address_list = self.getIPv4LocalAddressList()
    if addr is None:
      return self._generateRandomIPv4Address(netmask)
    elif dict(addr=addr, netmask=netmask) not in local_address_list:
        if self._addSystemAddress(addr, netmask, False):
          return dict(addr=addr, netmask=netmask)
        else:
          return self._generateRandomIPv4Address(netmask)
    else:
      # confirmed to be configured
      return dict(addr=addr, netmask=netmask)

  def addAddr(self, addr = None, netmask = None):
    """
    Adds IP address to bridge.

    If addr is specified and exists already on bridge does nothing.

    If addr is specified and does not exists on bridge, tries to add given address.
    In case if it is not possible (ex. because network changed) calculates new address.

    Args:
      addr: Wished address to be added to bridge.
      netmask: Wished netmask to be used.

    Returns:
      Tuple of (address, netmask).

    Raises:
      AddressGenerationError: Couldn't construct valid address with existing
          one's on the bridge.
      NoAddressOnBridge: There's no address on the bridge to construct
          an address with.
    """
    # Getting one address of the bridge as base of the next addresses
Łukasz Nowak's avatar
Łukasz Nowak committed
763 764 765 766
    if self.ipv6_interface:
      interface_name = self.ipv6_interface
    else:
      interface_name = self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
767 768 769 770
    bridge_addr_list = self.getGlobalScopeAddressList()

    # No address found
    if len(bridge_addr_list) == 0:
Łukasz Nowak's avatar
Łukasz Nowak committed
771
      raise NoAddressOnBridge(interface_name)
Łukasz Nowak's avatar
Łukasz Nowak committed
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
    address_dict = bridge_addr_list[0]

    if addr is not None:
      if dict(addr=addr, netmask=netmask) in bridge_addr_list:
        # confirmed to be configured
        return dict(addr=addr, netmask=netmask)
      if netmask == address_dict['netmask']:
        # same netmask, so there is a chance to add good one
        bridge_network = netaddr.ip.IPNetwork('%s/%s' % (address_dict['addr'],
          netmaskToPrefixIPv6(address_dict['netmask'])))
        requested_network = netaddr.ip.IPNetwork('%s/%s' % (addr, netmaskToPrefixIPv6(netmask)))
        if bridge_network.network == requested_network.network:
          # same network, try to add
          if self._addSystemAddress(addr, netmask):
            # succeed, return it
            return dict(addr=addr, netmask=netmask)

    # Try 10 times to add address, raise in case if not possible
    try_num = 10
    netmask = address_dict['netmask']
    while try_num > 0:
      addr = ':'.join(address_dict['addr'].split(':')[:-1] + ['%x' % random.randint(1, 65000)])
      socket.inet_pton(socket.AF_INET6, addr)
      if dict(addr=addr, netmask=netmask) not in self.getGlobalScopeAddressList():
        # Checking the validity of the IPv6 address
        if self._addSystemAddress(addr, netmask):
          return dict(addr=addr, netmask=netmask)
        try_num -= 1

    raise AddressGenerationError(addr)

class Parser(OptionParser):
  """
  Parse all arguments.
  """
  def __init__(self, usage=None, version=None):
    """
    Initialize all options possibles.
    """
    OptionParser.__init__(self, usage=usage, version=version,
                          option_list=[
      Option("-x", "--computer_xml",
             help="Path to file with computer's XML. If does not exists, "
                  "will be created",
             default=None,
             type=str),
      Option("-l", "--log_file",
             help="The path to the log file used by the script.",
             type=str),
      Option("-i", "--input_definition_file",
             help="Path to file to read definition of computer instead of "
             "declaration. Using definition file allows to disable "
             "'discovery' of machine services and allows to define computer "
             "configuration in fully controlled manner.",
             type=str),
      Option("-o", "--output_definition_file",
             help="Path to file to write definition of computer from "
             "declaration.",
             type=str),
      Option("-n", "--dry_run",
832 833 834
             help="Don't actually do anything.",
             default=False,
             action="store_true"),
Łukasz Nowak's avatar
Łukasz Nowak committed
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
      Option("-v", "--verbose",
             default=False,
             action="store_true",
             help="Verbose output."),
      Option("-c", "--console",
             default=False,
             action="store_true",
             help="Console output."),
      Option('--alter_user', choices=['True', 'False'],
        help="Shall slapformat alter user database [default: True]"),
      Option('--alter_network', choices=['True', 'False'],
        help="Shall slapformat alter network configuration [default: True]"),
      ])

  def check_args(self):
    """
    Check arguments
    """
    (options, args) = self.parse_args()
    if len(args) != 1:
      self.error("Incorrect number of arguments")
    return options, args[0]

def run(config):
  try:
    # Define the computer
    if config.input_definition_file:
      filepath = os.path.abspath(config.input_definition_file)
      config.logger.info('Using definition file %r' % filepath)
      computer_definition = ConfigParser.RawConfigParser()
      computer_definition.read(filepath)
      bridge = None
      address = None
      netmask = None
      if computer_definition.has_option('computer', 'address'):
        address, netmask = computer_definition.get('computer', 'address').split('/')
      if config.alter_network and config.bridge_name is not None \
          and config.ipv4_local_network is not None:
Łukasz Nowak's avatar
Łukasz Nowak committed
873 874
        bridge = Bridge(config.bridge_name, config.ipv4_local_network,
          config.ipv6_interface)
Łukasz Nowak's avatar
Łukasz Nowak committed
875 876 877 878 879
      computer = Computer(
          reference=config.computer_id,
          bridge=bridge,
          addr=address,
          netmask=netmask,
Łukasz Nowak's avatar
Łukasz Nowak committed
880
          ipv6_interface=config.ipv6_interface
Łukasz Nowak's avatar
Łukasz Nowak committed
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901
        )
      partition_list = []
      for partition_number in range(int(config.partition_amount)):
        section = 'partition_%s' % partition_number
        user = User(computer_definition.get(section, 'user'))
        address_list = []
        for a in computer_definition.get(section, 'address').split():
          address, netmask = a.split('/')
          address_list.append(dict(addr=address, netmask=netmask))
        tap = Tap(computer_definition.get(section, 'network_interface'))
        partition_list.append(Partition(reference=computer_definition.get(section, 'pathname'),
            path=os.path.join(config.instance_root, computer_definition.get(section, 'pathname')),
            user=user,
            address_list=address_list,
            tap=tap,
            ))
      computer.partition_list = partition_list
    else:
      # no definition file, figure out computer
      if os.path.exists(config.computer_xml):
        config.logger.info('Loading previous computer data from %r' % config.computer_xml)
Łukasz Nowak's avatar
Łukasz Nowak committed
902
        computer = Computer.load(config.computer_xml, reference=config.computer_id, ipv6_interface=config.ipv6_interface)
Łukasz Nowak's avatar
Łukasz Nowak committed
903
        # Connect to the bridge interface defined by the configuration
Łukasz Nowak's avatar
Łukasz Nowak committed
904 905
        computer.bridge = Bridge(config.bridge_name, config.ipv4_local_network,
            config.ipv6_interface)
Łukasz Nowak's avatar
Łukasz Nowak committed
906 907 908 909 910
      else:
        # If no pre-existent configuration found, creating a new computer object
        config.logger.warning('Creating new data computer with id %r' % config.computer_id)
        computer = Computer(
          reference=config.computer_id,
Łukasz Nowak's avatar
Łukasz Nowak committed
911 912
          bridge=Bridge(config.bridge_name, config.ipv4_local_network,
            config.ipv6_interface),
Łukasz Nowak's avatar
Łukasz Nowak committed
913 914
          addr=None,
          netmask=None,
Łukasz Nowak's avatar
Łukasz Nowak committed
915
          ipv6_interface=config.ipv6_interface
Łukasz Nowak's avatar
Łukasz Nowak committed
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
        )

      partition_amount = int(config.partition_amount)
      existing_partition_amount = len(computer.partition_list)
      if existing_partition_amount > partition_amount:
        raise ValueError('Requested amount of computer partitions (%s) is lower '
            'then already configured (%s), cannot continue' % (partition_amount,
              len(computer.partition_list)))

      config.logger.info('Adding %s new partitions' %
          (partition_amount-existing_partition_amount))
      for nb_iter in range(existing_partition_amount, partition_amount):
        # add new ones
        user = User("%s%s" % (config.user_base_name, nb_iter))

        tap = Tap("%s%s" % (config.tap_base_name, nb_iter))

        path = os.path.join(config.instance_root, "%s%s" % (
                             config.partition_base_name, nb_iter))
        computer.partition_list.append(
          Partition(
            reference="%s%s" % (config.partition_base_name, nb_iter),
            path=path,
            user=user,
            address_list=None,
            tap=tap,
            ))

    computer.instance_root = config.instance_root
    computer.software_root = config.software_root
    config.logger.info('Updating computer')
    address = computer.getAddress()
    computer.address = address['addr']
    computer.netmask = address['netmask']

    if config.output_definition_file:
      computer_definition = ConfigParser.RawConfigParser()
      computer_definition.add_section('computer')
      if computer.address is not None and computer.netmask is not None:
        computer_definition.set('computer', 'address', '/'.join([computer.address, computer.netmask]))
      partition_number = 0
      for partition in computer.partition_list:
        section = 'partition_%s' % partition_number
        computer_definition.add_section(section)
        address_list = []
        for address in partition.address_list:
          address_list.append('/'.join([address['addr'], address['netmask']]))
        computer_definition.set(section, 'address', ' '.join(address_list))
        computer_definition.set(section, 'user', partition.user.name)
        computer_definition.set(section, 'user', partition.user.name)
        computer_definition.set(section, 'network_interface', partition.tap.name)
        computer_definition.set(section, 'pathname', partition.reference)
        partition_number += 1
      filepath = os.path.abspath(config.output_definition_file)
      computer_definition.write(open(filepath, 'w'))
      config.logger.info('Stored computer definition in %r' % filepath)
    computer.construct(alter_user=config.alter_user,
        alter_network=config.alter_network)

    # Dumping and sending to the erp5 the current configuration
976 977
    if not config.dry_run:
      computer.dump(config.computer_xml)
Łukasz Nowak's avatar
Łukasz Nowak committed
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
    config.logger.info('Posting information to %r' % config.master_url)
    computer.send(config)
  except:
    config.logger.exception('Uncaught exception:')
    raise

class Config:
  def checkRequiredBinary(self, binary_list):
    missing_binary_list = []
    for b in binary_list:
      try:
        callAndRead([b])
      except ValueError:
        pass
      except OSError:
        missing_binary_list.append(b)
    if missing_binary_list:
      raise UsageError('Some required binaries are missing or not functional: %s'%
           ','.join(missing_binary_list))

  def setConfig(self, option_dict, configuration_file_path):
    """
    Set options given by parameters.
    """
    self.key_file = None
    self.cert_file = None
    # Set options parameters
    for option, value in option_dict.__dict__.items():
      setattr(self, option, value)

    # Load configuration file
    configuration_parser = ConfigParser.SafeConfigParser()
    configuration_parser.read(configuration_file_path)
    # Merges the arguments and configuration
    for section in ("slapformat", "slapos"):
      configuration_dict = dict(configuration_parser.items(section))
      for key in configuration_dict:
        if not getattr(self, key, None):
          setattr(self, key, configuration_dict[key])

    # setup some nones
    for parameter in ['bridge_name', 'partition_base_name', 'user_base_name',
Łukasz Nowak's avatar
Łukasz Nowak committed
1020
        'tap_base_name', 'ipv4_local_network', 'ipv6_interface']:
Łukasz Nowak's avatar
Łukasz Nowak committed
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
      if getattr(self, parameter, None) is None:
        setattr(self, parameter, None)

    # Set defaults lately
    if self.alter_network is None:
      self.alter_network = 'True'
    if self.alter_user is None:
      self.alter_user = 'True'

    # set up logging
    self.logger = logging.getLogger("slapformat")
    self.logger.setLevel(logging.INFO)
    if self.console:
      self.logger.addHandler(logging.StreamHandler())

    # Convert strings to booleans
    root_needed = False
    for o in ['alter_network', 'alter_user']:
      if getattr(self, o).lower() == 'true':
        root_needed = True
        setattr(self, o, True)
      elif getattr(self, o).lower() == 'false':
        setattr(self, o, False)
      else:
        message = 'Option %r needs to be "True" or "False", wrong value: %r' % (
            o, getattr(self, o))
        self.logger.error(message)
        raise UsageError(message)

1050 1051 1052 1053 1054 1055
    if not self.dry_run:
      if self.alter_user:
        self.checkRequiredBinary(['groupadd', 'useradd', 'usermod'])
      if self.alter_network:
        self.checkRequiredBinary(['ip', 'tunctl'])
    # Required, even for dry run
Łukasz Nowak's avatar
Łukasz Nowak committed
1056
    if self.alter_network:
1057
      self.checkRequiredBinary(['brctl'])
Łukasz Nowak's avatar
Łukasz Nowak committed
1058

1059 1060
    if self.dry_run:
      root_needed = False
Łukasz Nowak's avatar
Łukasz Nowak committed
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
    
    # check root
    if root_needed and os.getuid() != 0:
      message = "Root rights are needed"
      self.logger.error(message)
      raise UsageError(message)

    if self.log_file:
      if not os.path.isdir(os.path.dirname(self.log_file)):
        # fallback to console only if directory for logs does not exists and
        # continue to run
        raise ValueError('Please create directory %r to store %r log file' % (
          os.path.dirname(self.log_file), self.log_file))
      else:
        file_handler = logging.FileHandler(self.log_file)
        file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
        self.logger.addHandler(file_handler)
        self.logger.info('Configured logging to file %r' % self.log_file)
    # Check mandatory options
    for parameter in ('computer_id', 'instance_root', 'master_url',
                      'software_root', 'computer_xml'):
      if not getattr(self, parameter, None):
        raise UsageError("Parameter '%s' is not defined." % parameter)

    self.logger.info("Started.")
    if self.verbose:
      self.logger.setLevel(logging.DEBUG)
      self.logger.debug("Verbose mode enabled.")
1089 1090
    if self.dry_run:
      self.logger.info("Dry-run mode enabled.")
Łukasz Nowak's avatar
Łukasz Nowak committed
1091 1092 1093 1094 1095 1096 1097

    # Calculate path once
    self.computer_xml = os.path.abspath(self.computer_xml)


def main():
  "Run default configuration."
Vincent Pelletier's avatar
Vincent Pelletier committed
1098 1099
  global os
  global callAndRead
1100
  global getpwnam
Vincent Pelletier's avatar
Vincent Pelletier committed
1101
  real_callAndRead = callAndRead
Łukasz Nowak's avatar
Łukasz Nowak committed
1102 1103 1104 1105 1106 1107 1108
  usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]

  try:
    # Parse arguments
    options, configuration_file_path = Parser(usage=usage).check_args()
    config = Config()
    config.setConfig(options, configuration_file_path)
Vincent Pelletier's avatar
Vincent Pelletier committed
1109
    os = OS(config)
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
    if config.dry_run:
      def dry_callAndRead(argument_list, raise_on_error=True):
        if argument_list == ['brctl', 'show']:
          return real_callAndRead(argument_list, raise_on_error)
        else:
          return 0, ''
      callAndRead = dry_callAndRead
      real_addSystemAddress = Bridge._addSystemAddress
      def fake_addSystemAddress(*args, **kw):
        real_addSystemAddress(*args, **kw)
        # Fake success
        return True
      Bridge._addSystemAddress = fake_addSystemAddress
      def fake_getpwnam(user):
        class result:
          pw_uid = 12345
          pw_gid = 54321
        return result
      getpwnam = fake_getpwnam
    else:
      dry_callAndRead = real_callAndRead
Vincent Pelletier's avatar
Vincent Pelletier committed
1131 1132 1133
    if config.verbose:
      def logging_callAndRead(argument_list, raise_on_error=True):
        config.logger.debug(' '.join(argument_list))
1134
        return dry_callAndRead(argument_list, raise_on_error)
Vincent Pelletier's avatar
Vincent Pelletier committed
1135
      callAndRead = logging_callAndRead
Łukasz Nowak's avatar
Łukasz Nowak committed
1136 1137 1138 1139
    run(config)
  except UsageError, err:
    print >>sys.stderr, err.msg
    print >>sys.stderr, "For help use --help"