format.py 57.7 KB
Newer Older
Łukasz Nowak's avatar
Łukasz Nowak committed
1
# -*- coding: utf-8 -*-
Marco Mariani's avatar
Marco Mariani committed
2
# vim: set et sts=2:
Łukasz Nowak's avatar
Łukasz Nowak committed
3 4
##############################################################################
#
5 6
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
Łukasz Nowak's avatar
Łukasz Nowak committed
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
30

Bryton Lacquement's avatar
Bryton Lacquement committed
31
from six.moves import configparser
32 33
import errno
import fcntl
Łukasz Nowak's avatar
Łukasz Nowak committed
34
import grp
35
import json
Łukasz Nowak's avatar
Łukasz Nowak committed
36
import logging
37
import math
Łukasz Nowak's avatar
Łukasz Nowak committed
38 39 40
import netaddr
import netifaces
import os
41
import glob
Łukasz Nowak's avatar
Łukasz Nowak committed
42
import pwd
Łukasz Nowak's avatar
Łukasz Nowak committed
43
import random
44
import shutil
Łukasz Nowak's avatar
Łukasz Nowak committed
45
import socket
46
import struct
Łukasz Nowak's avatar
Łukasz Nowak committed
47 48
import subprocess
import sys
49
import threading
Łukasz Nowak's avatar
Łukasz Nowak committed
50
import time
51
import traceback
Marco Mariani's avatar
Marco Mariani committed
52
import zipfile
53
import platform
Bryton Lacquement's avatar
Bryton Lacquement committed
54 55
from six.moves.urllib.request import urlopen
import six
Marco Mariani's avatar
Marco Mariani committed
56 57

import lxml.etree
58
import xml_marshaller.xml_marshaller
Marco Mariani's avatar
Marco Mariani committed
59

Bryton Lacquement's avatar
Bryton Lacquement committed
60
from slapos.util import dumps, mkdir_p, ipv6FromBin, binFromIpv6, lenNetmaskIpv6
61
import slapos.slap as slap
62
from slapos import version
63
from slapos import manager as slapmanager
64

65 66 67 68

logger = logging.getLogger("slapos.format")


Marco Mariani's avatar
Marco Mariani committed
69 70 71 72
def prettify_xml(xml):
  root = lxml.etree.fromstring(xml)
  return lxml.etree.tostring(root, pretty_print=True)

Łukasz Nowak's avatar
Łukasz Nowak committed
73

Vincent Pelletier's avatar
Vincent Pelletier committed
74
class OS(object):
75 76
  """Wrap parts of the 'os' module to provide logging of performed actions."""

Vincent Pelletier's avatar
Vincent Pelletier committed
77 78
  _os = os

79 80 81
  def __init__(self, conf):
    self._dry_run = conf.dry_run
    self._logger = conf.logger
Vincent Pelletier's avatar
Vincent Pelletier committed
82 83 84 85 86 87 88 89
    add = self._addWrapper
    add('chown')
    add('chmod')
    add('makedirs')
    add('mkdir')

  def _addWrapper(self, name):
    def wrapper(*args, **kw):
90
      arg_list = [repr(x) for x in args] + [
Bryton Lacquement's avatar
Bryton Lacquement committed
91
          '%s=%r' % (x, y) for x, y in six.iteritems(kw)
92
      ]
93
      self._logger.debug('%s(%s)' % (name, ', '.join(arg_list)))
94 95
      if not self._dry_run:
        getattr(self._os, name)(*args, **kw)
Vincent Pelletier's avatar
Vincent Pelletier committed
96 97 98 99
    setattr(self, name, wrapper)

  def __getattr__(self, name):
    return getattr(self._os, name)
Łukasz Nowak's avatar
Łukasz Nowak committed
100

101

102 103 104
class UsageError(Exception):
  pass

105

106
class NoAddressOnInterface(Exception):
Łukasz Nowak's avatar
Łukasz Nowak committed
107
  """
Marco Mariani's avatar
Marco Mariani committed
108
  Exception raised if there is no address on the interface to construct IPv6
109 110 111
  address with.

  Attributes:
112
    brige: String, the name of the interface.
Łukasz Nowak's avatar
Łukasz Nowak committed
113 114
  """

115 116
  def __init__(self, interface):
    super(NoAddressOnInterface, self).__init__(
Marco Mariani's avatar
Marco Mariani committed
117
      'No IPv6 found on interface %s to construct IPv6 with.' % interface
118
    )
Łukasz Nowak's avatar
Łukasz Nowak committed
119

120

121 122 123
class AddressGenerationError(Exception):
  """
  Exception raised if the generation of an IPv6 based on the prefix obtained
124
  from the interface failed.
125 126 127 128 129 130 131 132

  Attributes:
    addr: String, the invalid address the exception is raised for.
  """
  def __init__(self, addr):
    super(AddressGenerationError, self).__init__(
      'Generated IPv6 %s seems not to be a valid IP.' % addr
    )
Łukasz Nowak's avatar
Łukasz Nowak committed
133

134

135 136 137 138 139 140 141 142 143 144 145
def getPublicIPv4Address():
  test_list = [
    { "url": 'https://api.ipify.org/?format=json' , "json_key": "ip"},
    { "url": 'http://httpbin.org/ip', "json_key": "origin"},
    { "url": 'http://jsonip.com', "json_key": "ip"}]
  previous = None
  ipv4 = None
  for test in test_list:
    if ipv4 is not None:
      previous = ipv4
    try:
146
      ipv4 = json.load(urlopen(test["url"], timeout=15))[test["json_key"]]
147 148 149 150 151
    except:
      ipv4 = None
    if ipv4 is not None and ipv4 == previous:
      return ipv4

Łukasz Nowak's avatar
Łukasz Nowak committed
152
def callAndRead(argument_list, raise_on_error=True):
Marco Mariani's avatar
Marco Mariani committed
153 154 155
  popen = subprocess.Popen(argument_list,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
Łukasz Nowak's avatar
Łukasz Nowak committed
156 157
  result = popen.communicate()[0]
  if raise_on_error and popen.returncode != 0:
158
    raise ValueError('Issue while invoking %r, result was:\n%s' % (
Marco Mariani's avatar
Marco Mariani committed
159
                     argument_list, result))
Łukasz Nowak's avatar
Łukasz Nowak committed
160 161
  return popen.returncode, result

162

Łukasz Nowak's avatar
Łukasz Nowak committed
163 164 165 166 167 168
def isGlobalScopeAddress(a):
  """Returns True if a is global scope IP v4/6 address"""
  ip = netaddr.IPAddress(a)
  return not ip.is_link_local() and not ip.is_loopback() and \
      not ip.is_reserved() and ip.is_unicast()

169

Łukasz Nowak's avatar
Łukasz Nowak committed
170 171 172 173 174
def netmaskToPrefixIPv4(netmask):
  """Convert string represented netmask to its integer prefix"""
  return netaddr.strategy.ipv4.netmask_to_prefix[
          netaddr.strategy.ipv4.str_to_int(netmask)]

175
def getIfaceAddressIPv4(iface):
176
  """return dict containing ipv4 address netmask, network and broadcast address
177 178 179 180 181 182 183
  of interface"""
  if not iface in netifaces.interfaces():
    raise ValueError('Could not find interface called %s to use as gateway ' \
                      'for tap network' % iface)
  try:
    addresses_list = netifaces.ifaddresses(iface)[socket.AF_INET]
    if len (addresses_list) > 0:
184

185 186 187 188 189 190 191 192 193 194
      addresses = addresses_list[0].copy()
      addresses['network'] = str(netaddr.IPNetwork('%s/%s' % (addresses['addr'],
                                          addresses['netmask'])).cidr.network)
      return addresses
    else:
      return {}
  except KeyError:
    raise KeyError('Could not find IPv4 adress on interface %s.' % iface)

def getIPv4SubnetAddressRange(ip_address, mask, size):
195
  """Check if a given ipaddress can be used to create 'size'
196 197 198
  host ip address, then return list of ip address in the subnet"""
  ip = netaddr.IPNetwork('%s/%s' % (ip_address, mask))
  # Delete network and default ip_address from the list
199
  ip_list = [x for x in sorted(list(ip))
200 201 202 203 204
              if str(x) != ip_address and x.value != ip.cidr.network.value]
  if len(ip_list) < size:
    raise ValueError('Could not create %s tap interfaces from address %s.' % (
              size, ip_address))
  return ip_list
205

206
def _getDict(obj):
Łukasz Nowak's avatar
Łukasz Nowak committed
207
  """
208
  Serialize an object into dictionaries. List and dict will remains
Łukasz Nowak's avatar
Łukasz Nowak committed
209 210 211 212
  the same, basic type too. But encapsulated object will be returned as dict.
  Set, collections and other aren't handle for now.

  Args:
213
    obj: an object of any type.
Łukasz Nowak's avatar
Łukasz Nowak committed
214 215 216 217

  Returns:
    A dictionary if the given object wasn't a list, a list otherwise.
  """
218 219
  if isinstance(obj, list):
    return [_getDict(item) for item in obj]
Łukasz Nowak's avatar
Łukasz Nowak committed
220

221 222
  if isinstance(obj, dict):
    dikt = obj
Łukasz Nowak's avatar
Łukasz Nowak committed
223 224
  else:
    try:
225
      dikt = obj.__dict__
Łukasz Nowak's avatar
Łukasz Nowak committed
226
    except AttributeError:
227 228 229
      return obj

  return {
Bryton Lacquement's avatar
Bryton Lacquement committed
230 231
    key: _getDict(value)
    for key, value in six.iteritems(dikt)
232
    # do not attempt to serialize logger: it is both useless and recursive.
233 234
    # do not serialize attributes starting with "_", let the classes have some privacy
    if not key.startswith("_")
235
  }
Łukasz Nowak's avatar
Łukasz Nowak committed
236

237

238
class Computer(object):
239
  """Object representing the computer"""
Łukasz Nowak's avatar
Łukasz Nowak committed
240

241
  def __init__(self, reference, interface=None, addr=None, netmask=None,
242
               ipv6_interface=None, software_user='slapsoft',
243
               tap_gateway_interface=None, tap_ipv6=None,
244
               instance_root=None, software_root=None, instance_storage_home=None,
245
               partition_list=None, config=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
246 247
    """
    Attributes:
248 249
      reference: str, the reference of the computer.
      interface: str, the name of the computer's used interface.
250 251

      :param config: dict-like, holds raw data from configuration file
Łukasz Nowak's avatar
Łukasz Nowak committed
252 253
    """
    self.reference = str(reference)
254
    self.interface = interface
255
    self.partition_list = partition_list or []
Łukasz Nowak's avatar
Łukasz Nowak committed
256 257
    self.address = addr
    self.netmask = netmask
258
    self.ipv6_interface = ipv6_interface
259
    self.software_user = software_user
260
    self.tap_gateway_interface = tap_gateway_interface
261
    self.tap_ipv6 = tap_ipv6
Łukasz Nowak's avatar
Łukasz Nowak committed
262

263 264 265 266 267 268 269 270
    # Used to be static attributes of the class object - didn't make sense (Marco again)
    assert instance_root is not None and software_root is not None, \
           "Computer's instance_root and software_root must not be empty!"
    self.software_root = software_root
    self.instance_root = instance_root
    self.instance_storage_home = instance_storage_home

    # The following properties are updated on update() method
271 272 273 274 275
    self.public_ipv4_address = None
    self.os_type = None
    self.python_version = None
    self.slapos_version = None

276 277
    # attributes starting with '_' are saved from serialization
    # monkey-patch use of class instead of dictionary
278 279 280 281
    if config is None:
      logger.warning("Computer needs config in constructor to allow managers.")

    self._config = config if config is None or isinstance(config, dict) else config.__dict__
282
    self._manager_list = slapmanager.from_config(self._config)
283

Łukasz Nowak's avatar
Łukasz Nowak committed
284
  def __getinitargs__(self):
285
    return (self.reference, self.interface)
Łukasz Nowak's avatar
Łukasz Nowak committed
286

287
  def getAddress(self):
Łukasz Nowak's avatar
Łukasz Nowak committed
288
    """
Marco Mariani's avatar
Marco Mariani committed
289
    Return a list of the interface address not attributed to any partition (which
Łukasz Nowak's avatar
Łukasz Nowak committed
290 291 292
    are therefore free for the computer itself).

    Returns:
293
      False if the interface isn't available, else the list of the free addresses.
Łukasz Nowak's avatar
Łukasz Nowak committed
294
    """
295
    if self.interface is None:
Marco Mariani's avatar
Marco Mariani committed
296
      return {'addr': self.address, 'netmask': self.netmask}
Łukasz Nowak's avatar
Łukasz Nowak committed
297 298 299 300 301 302

    computer_partition_address_list = []
    for partition in self.partition_list:
      for address in partition.address_list:
        if netaddr.valid_ipv6(address['addr']):
          computer_partition_address_list.append(address['addr'])
303
    # Going through addresses of the computer's interface
304 305 306 307 308 309 310 311 312 313
    available_address_list = self.interface.getGlobalScopeAddressList()
    # First scan: prefer the existing address, as the available_address_list
    #             might be sorted differently
    for address_dict in available_address_list:
      if self.address == address_dict['addr']:
        if self.address not in computer_partition_address_list:
          return address_dict

    # Second scan: take first not occupied address
    for address_dict in available_address_list:
Łukasz Nowak's avatar
Łukasz Nowak committed
314 315 316 317
      # Comparing with computer's partition addresses
      if address_dict['addr'] not in computer_partition_address_list:
        return address_dict

318
    # Can't find address
Marco Mariani's avatar
Marco Mariani committed
319
    raise NoAddressOnInterface('No valid IPv6 found on %s.' % self.interface.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
320

321
  def update(self):
322
    """Collect environmental hardware/network information."""
323 324 325 326 327
    self.public_ipv4_address = getPublicIPv4Address()
    self.slapos_version = version.version
    self.python_version = platform.python_version()
    self.os_type = platform.platform()

328
  def send(self, conf):
Łukasz Nowak's avatar
Łukasz Nowak committed
329 330 331 332 333
    """
    Send a marshalled dictionary of the computer object serialized via_getDict.
    """
    slap_instance = slap.slap()
    connection_dict = {}
334 335 336 337
    if conf.key_file and conf.cert_file:
      connection_dict['key_file'] = conf.key_file
      connection_dict['cert_file'] = conf.cert_file
    slap_instance.initializeConnection(conf.master_url,
Marco Mariani's avatar
Marco Mariani committed
338
                                       **connection_dict)
Łukasz Nowak's avatar
Łukasz Nowak committed
339
    slap_computer = slap_instance.registerComputer(self.reference)
Marco Mariani's avatar
Marco Mariani committed
340

341
    if conf.dry_run:
342
      return
343
    try:
Bryton Lacquement's avatar
Bryton Lacquement committed
344
      slap_computer.updateConfiguration(dumps(_getDict(self)))
345
    except slap.NotFoundError as error:
346 347 348
      raise slap.NotFoundError("%s\nERROR: This SlapOS node is not recognised by "
          "SlapOS Master and/or computer_id and certificates don't match. "
          "Please make sure computer_id of slapos.cfg looks "
349
          "like 'COMP-123' and is correct.\nError is : 404 Not Found." % error)
Łukasz Nowak's avatar
Łukasz Nowak committed
350

351
  def dump(self, path_to_xml, path_to_json, logger):
Łukasz Nowak's avatar
Łukasz Nowak committed
352 353 354 355 356
    """
    Dump the computer object to an xml file via xml_marshaller.

    Args:
      path_to_xml: String, path to the file to load.
Marco Mariani's avatar
Marco Mariani committed
357
      path_to_json: String, path to the JSON version to save.
Łukasz Nowak's avatar
Łukasz Nowak committed
358 359
    """

360 361 362 363
    # dump partition resources information
    for partition in self.partition_list:
      partition.dump()

Łukasz Nowak's avatar
Łukasz Nowak committed
364
    computer_dict = _getDict(self)
365 366 367 368 369

    if path_to_json:
      with open(path_to_json, 'wb') as fout:
        fout.write(json.dumps(computer_dict, sort_keys=True, indent=2))

Bryton Lacquement's avatar
Bryton Lacquement committed
370
    new_xml = dumps(computer_dict)
Marco Mariani's avatar
Marco Mariani committed
371 372 373 374
    new_pretty_xml = prettify_xml(new_xml)

    path_to_archive = path_to_xml + '.zip'

375
    if os.path.exists(path_to_archive) and os.path.exists(path_to_xml):
Marco Mariani's avatar
Marco Mariani committed
376 377 378 379 380 381
      # the archive file exists, we only backup if something has changed
      with open(path_to_xml, 'rb') as fin:
        if fin.read() == new_pretty_xml:
          # computer configuration did not change, nothing to write
          return

382
    if os.path.exists(path_to_xml):
383 384 385 386 387
      try:
        self.backup_xml(path_to_archive, path_to_xml)
      except:
        # might be a corrupted zip file. let's move it out of the way and retry.
        shutil.move(path_to_archive,
Marco Mariani's avatar
Marco Mariani committed
388
                    path_to_archive + time.strftime('_broken_%Y%m%d-%H:%M'))
389 390 391 392
        try:
          self.backup_xml(path_to_archive, path_to_xml)
        except:
          # give up trying
393
          logger.exception("Can't backup %s:", path_to_xml)
Marco Mariani's avatar
Marco Mariani committed
394

Marco Mariani's avatar
Marco Mariani committed
395 396
    with open(path_to_xml, 'wb') as fout:
      fout.write(new_pretty_xml)
Marco Mariani's avatar
Marco Mariani committed
397 398

  def backup_xml(self, path_to_archive, path_to_xml):
Marco Mariani's avatar
Marco Mariani committed
399 400 401
    """
    Stores a copy of the current xml file to an historical archive.
    """
Marco Mariani's avatar
Marco Mariani committed
402
    xml_content = open(path_to_xml).read()
Marco Mariani's avatar
Marco Mariani committed
403
    saved_filename = os.path.basename(path_to_xml) + time.strftime('.%Y%m%d-%H:%M')
Marco Mariani's avatar
Marco Mariani committed
404 405 406 407

    with zipfile.ZipFile(path_to_archive, 'a') as archive:
      archive.writestr(saved_filename, xml_content, zipfile.ZIP_DEFLATED)

Łukasz Nowak's avatar
Łukasz Nowak committed
408
  @classmethod
409
  def load(cls, path_to_xml, reference, ipv6_interface, tap_gateway_interface,
410
           tap_ipv6, instance_root=None, software_root=None, config=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
411 412 413 414 415 416 417 418
    """
    Create a computer object from a valid xml file.

    Arg:
      path_to_xml: String, a path to a valid file containing
          a valid configuration.

    Return:
419
      A Computer object.
Łukasz Nowak's avatar
Łukasz Nowak committed
420
    """
421 422
    with open(path_to_xml, "rb") as fi:
      dumped_dict = xml_marshaller.xml_marshaller.load(fi)
Łukasz Nowak's avatar
Łukasz Nowak committed
423 424 425

    # Reconstructing the computer object from the xml
    computer = Computer(
426 427 428 429 430
        reference=reference,
        addr=dumped_dict['address'],
        netmask=dumped_dict['netmask'],
        ipv6_interface=ipv6_interface,
        software_user=dumped_dict.get('software_user', 'slapsoft'),
431
        tap_gateway_interface=tap_gateway_interface,
432
        tap_ipv6=tap_ipv6,
433 434
        software_root=dumped_dict.get('software_root', software_root),
        instance_root=dumped_dict.get('instance_root', instance_root),
435
        config=config,
Łukasz Nowak's avatar
Łukasz Nowak committed
436 437
    )

438
    partition_amount = int(config.partition_amount)
439
    for partition_index, partition_dict in enumerate(dumped_dict['partition_list']):
Łukasz Nowak's avatar
Łukasz Nowak committed
440 441 442 443 444 445 446 447

      if partition_dict['user']:
        user = User(partition_dict['user']['name'])
      else:
        user = User('root')

      if partition_dict['tap']:
        tap = Tap(partition_dict['tap']['name'])
448 449 450 451
        tap.ipv4_addr = partition_dict['tap'].get('ipv4_addr', '')
        tap.ipv4_netmask = partition_dict['tap'].get('ipv4_netmask', '')
        tap.ipv4_gateway = partition_dict['tap'].get('ipv4_gateway', '')
        tap.ipv4_network = partition_dict['tap'].get('ipv4_network', '')
452 453 454 455
        tap.ipv6_addr = partition_dict['tap'].get('ipv6_addr', '')
        tap.ipv6_netmask = partition_dict['tap'].get('ipv6_netmask', '')
        tap.ipv6_gateway = partition_dict['tap'].get('ipv6_gateway', '')
        tap.ipv6_network = partition_dict['tap'].get('ipv6_network', '')
Łukasz Nowak's avatar
Łukasz Nowak committed
456 457 458
      else:
        tap = Tap(partition_dict['reference'])

459
      if partition_dict.get('tun') is not None and partition_dict['tun'].get('ipv4_addr') is not None:
460
        tun = Tun(partition_dict['tun']['name'], partition_index, partition_amount)
461 462
        tun.ipv4_addr = partition_dict['tun']['ipv4_addr']
      else:
463
        tun = Tun("slaptun" + str(partition_index), partition_index, partition_amount)
464

Łukasz Nowak's avatar
Łukasz Nowak committed
465
      address_list = partition_dict['address_list']
466
      external_storage_list = partition_dict.get('external_storage_list', [])
Łukasz Nowak's avatar
Łukasz Nowak committed
467 468

      partition = Partition(
469 470 471 472 473
          reference=partition_dict['reference'],
          path=partition_dict['path'],
          user=user,
          address_list=address_list,
          tap=tap,
474
          tun=tun if config.create_tun else None,
475
          external_storage_list=external_storage_list,
Łukasz Nowak's avatar
Łukasz Nowak committed
476 477 478 479 480 481
      )

      computer.partition_list.append(partition)

    return computer

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
  def _speedHackAddAllOldIpsToInterface(self):
    """
    Speed hack:
    Blindly add all IPs from existing configuration, just to speed up actual
    computer configuration later on.
    """
    # XXX-TODO: only add an address if it doesn't already exist.
    if self.ipv6_interface:
      interface_name = self.ipv6_interface
    elif self.interface:
      interface_name = self.interface.name
    else:
      return

    for partition in self.partition_list:
      try:
        for address in partition.address_list:
          try:
500
            netmask = lenNetmaskIpv6(address['netmask'])
501 502 503 504 505 506 507 508
          except:
            continue
          callAndRead(['ip', 'addr', 'add',
                       '%s/%s' % (address['addr'], netmask),
                       'dev', interface_name])
      except ValueError:
        pass

509 510 511 512 513 514 515
  def _addUniqueLocalAddressIpv6(self, interface_name):
    """
    Create a unique local address in the interface interface_name, so that
    slapformat can build upon this.
    See https://en.wikipedia.org/wiki/Unique_local_address.
    """
    command = 'ip address add dev %s fd00::1/64' % interface_name
516
    callAndRead(command.split())
517

518 519 520 521
  @property
  def software_gid(self):
    """Return GID for self.software_user.

522
    Has to be dynamic because __init__ happens before ``format`` where we
523 524 525 526
    effectively create the user and group."""
    return pwd.getpwnam(self.software_user)[3]

  def format(self, alter_user=True, alter_network=True, create_tap=True, use_unique_local_address_block=False):
Łukasz Nowak's avatar
Łukasz Nowak committed
527
    """
528 529 530 531 532 533
    Setup underlaying OS so it reflects this instance (``self``).

    - setup interfaces and addresses
    - setup TAP and TUN interfaces
    - add groups and users
    - construct partitions inside slapgrid
Łukasz Nowak's avatar
Łukasz Nowak committed
534
    """
535
    for path in self.instance_root, self.software_root:
Łukasz Nowak's avatar
Łukasz Nowak committed
536
      if not os.path.exists(path):
Marco Mariani's avatar
Marco Mariani committed
537
        os.makedirs(path, 0o755)
Łukasz Nowak's avatar
Łukasz Nowak committed
538
      else:
Marco Mariani's avatar
Marco Mariani committed
539
        os.chmod(path, 0o755)
Łukasz Nowak's avatar
Łukasz Nowak committed
540

541 542
    # own self.software_root by software user
    slapsoft = User(self.software_user)
Łukasz Nowak's avatar
Łukasz Nowak committed
543 544 545
    slapsoft.path = self.software_root
    if alter_user:
      slapsoft.create()
Łukasz Nowak's avatar
Łukasz Nowak committed
546
      slapsoft_pw = pwd.getpwnam(slapsoft.name)
547
      os.chown(slapsoft.path, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
Marco Mariani's avatar
Marco Mariani committed
548
    os.chmod(self.software_root, 0o755)
Łukasz Nowak's avatar
Łukasz Nowak committed
549

550 551
    # Iterate over all managers and let them `format` the computer too
    for manager in self._manager_list:
552
      manager.format(self)
553

554 555 556
    # get list of instance external storage if exist
    instance_external_list = []
    if self.instance_storage_home:
557 558 559 560 561 562 563
      # get all /XXX/dataN where N is a digit
      data_list = glob.glob(os.path.join(self.instance_storage_home, 'data*'))
      for i in range(0, len(data_list)):
        data_path = data_list.pop()
        the_digit = os.path.basename(data_path).split('data')[-1]
        if the_digit.isdigit():
          instance_external_list.append(data_path)
564

565 566 567 568 569 570
    ####################
    ### Network part ###
    ####################
    if alter_network:
      if self.address is not None:
        self.interface.addIPv6Address(self.address, self.netmask)
571

572
      if use_unique_local_address_block:
573
        self._addUniqueLocalAddressIpv6(self.ipv6_interface or self.interface.name)
574 575 576 577 578 579 580 581 582 583 584 585 586 587

      if create_tap:
        if self.tap_gateway_interface:
          gateway_addr_dict = getIfaceAddressIPv4(self.tap_gateway_interface)
          tap_address_list = getIPv4SubnetAddressRange(gateway_addr_dict['addr'],
                                gateway_addr_dict['netmask'],
                                len(self.partition_list))
          assert(len(self.partition_list) <= len(tap_address_list))
        else:
          gateway_addr_dict = {'peer': '10.0.0.1', 'netmask': '255.255.0.0',
                               'addr': '10.0.0.1', 'network': '10.0.0.0'}
          tap_address_list = getIPv4SubnetAddressRange(gateway_addr_dict['addr'],
                                 gateway_addr_dict['netmask'],
                                 len(self.partition_list))
588

589
      self._speedHackAddAllOldIpsToInterface()
590

591 592 593 594 595 596
    try:
      for partition_index, partition in enumerate(self.partition_list):
        # Reconstructing User's
        partition.path = os.path.join(self.instance_root, partition.reference)
        partition.user.setPath(partition.path)
        partition.user.additional_group_list = [slapsoft.name]
597 598
        partition.external_storage_list = ['%s/%s' % (path, partition.reference)
                                            for path in instance_external_list]
599 600 601
        if alter_user:
          partition.user.create()

602 603 604 605 606 607 608 609 610 611 612
        # Reconstructing Tap and Tun
        if alter_network:
          if partition.user and partition.user.isAvailable():
            owner = partition.user
          else:
            owner = User('root')

          if create_tap:
            partition.tap.createWithOwner(owner)

            # add addresses and create route for this tap
613 614 615
            # Pop IP from tap_address_list and attach to tap if has no ipv4 yet
            next_ipv4_addr = '%s' % tap_address_list.pop(0)
            # skip to set this IP to tap if already exits
616 617 618 619 620 621 622
            if not partition.tap.ipv4_addr:
              # define new ipv4 address for this tap
              partition.tap.ipv4_addr = next_ipv4_addr
              partition.tap.ipv4_netmask = gateway_addr_dict['netmask']
              partition.tap.ipv4_gateway = gateway_addr_dict['addr']
              partition.tap.ipv4_network = gateway_addr_dict['network']

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639
            if self.tap_ipv6:
              if not partition.tap.ipv6_addr:
                # create a new IPv6 randomly for the tap
                ipv6_dict = self.interface.addIPv6Address(tap=partition.tap)
                partition.tap.ipv6_addr = ipv6_dict['addr']
                partition.tap.ipv6_netmask = ipv6_dict['netmask']
              else:
                # make sure the tap has its IPv6
                self.interface.addIPv6Address(
                        addr=partition.tap.ipv6_addr,
                        netmask=partition.tap.ipv6_netmask,
                        tap=partition.tap)

              # construct ipv6_network (16 bit more than the computer network)
              netmask_len = lenNetmaskIpv6(self.interface.getGlobalScopeAddressList()[0]['netmask']) + 16
              prefix = binFromIpv6(partition.tap.ipv6_addr)[:netmask_len]
              network_addr = ipv6FromBin(prefix)
640 641
              partition.tap.ipv6_gateway = "{}1".format(network_addr) # address network::1 will be inside the VM
              partition.tap.ipv6_gateway = ipv6FromBin(binFromIpv6(partition.tap.ipv6_gateway)) # correctly format the IPv6
642
              partition.tap.ipv6_network = "{}/{}".format(network_addr, netmask_len)
643
            else:
644 645 646 647 648 649
              partition.tap.ipv6_addr = ''
              partition.tap.ipv6_netmask = ''
              partition.tap.ipv6_gateway = ''
              partition.tap.ipv6_network = ''

            # create IPv4 and IPv6 routes
650 651 652 653 654 655
            partition.tap.createRoutes()

          if partition.tun is not None:
            # create TUN interface per partition as well
            partition.tun.createWithOwner(owner)
            partition.tun.createRoutes()
656

657 658
        # Reconstructing partition's directory
        partition.createPath(alter_user)
659
        partition.createExternalPath(alter_user)
660 661 662 663

        # Reconstructing partition's address
        # There should be two addresses on each Computer Partition:
        #  * local IPv4, took from slapformat:ipv4_local_network
664
        #  * global IPv6
Marco Mariani's avatar
Marco Mariani committed
665
        if not partition.address_list:
666
          # regenerate
667
          partition.address_list.append(self.interface.addIPv4LocalAddress())
668
          partition.address_list.append(self.interface.addIPv6Address())
669 670 671 672 673
        elif alter_network:
          # regenerate list of addresses
          old_partition_address_list = partition.address_list
          partition.address_list = []
          if len(old_partition_address_list) != 2:
674 675 676
            raise ValueError(
              'There should be exactly 2 stored addresses. Got: %r' %
              (old_partition_address_list,))
Marco Mariani's avatar
Marco Mariani committed
677 678
          if not any(netaddr.valid_ipv6(q['addr'])
                     for q in old_partition_address_list):
679
            raise ValueError('Not valid ipv6 addresses loaded')
Marco Mariani's avatar
Marco Mariani committed
680 681
          if not any(netaddr.valid_ipv4(q['addr'])
                     for q in old_partition_address_list):
682
            raise ValueError('Not valid ipv6 addresses loaded')
Marco Mariani's avatar
Marco Mariani committed
683

684 685
          for address in old_partition_address_list:
            if netaddr.valid_ipv6(address['addr']):
686
              partition.address_list.append(self.interface.addIPv6Address(
Vincent Pelletier's avatar
Vincent Pelletier committed
687 688
                address['addr'],
                address['netmask']))
689
            elif netaddr.valid_ipv4(address['addr']):
690
              partition.address_list.append(self.interface.addIPv4LocalAddress(
Vincent Pelletier's avatar
Vincent Pelletier committed
691
                address['addr']))
692 693 694
            else:
              raise ValueError('Address %r is incorrect' % address['addr'])
    finally:
695 696 697
      for manager in self._manager_list:
        manager.formatTearDown(self)

698

699
class Partition(object):
700 701 702
  """Represent a computer partition."""

  resource_file = ".slapos-resource"
Łukasz Nowak's avatar
Łukasz Nowak committed
703

704
  def __init__(self, reference, path, user, address_list,
705
               tap, external_storage_list=[], tun=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
706 707 708 709 710 711
    """
    Attributes:
      reference: String, the name of the partition.
      path: String, the path to the partition folder.
      user: User, the user linked to this partition.
      address_list: List of associated IP addresses.
712
      tap: Tap, the tap interface linked to this partition e.g. used as a gateway for kvm
713
      tun: Tun interface used for special apps simulating ethernet connections
714
      external_storage_list: Base path list of folder to format for data storage
Łukasz Nowak's avatar
Łukasz Nowak committed
715 716 717 718 719 720 721
    """

    self.reference = str(reference)
    self.path = str(path)
    self.user = user
    self.address_list = address_list or []
    self.tap = tap
722
    self.tun = tun
723
    self.external_storage_list = []
Łukasz Nowak's avatar
Łukasz Nowak committed
724 725

  def __getinitargs__(self):
726
    return (self.reference, self.path, self.user, self.address_list, self.tap, self.tun)
Łukasz Nowak's avatar
Łukasz Nowak committed
727 728 729

  def createPath(self, alter_user=True):
    """
Vincent Pelletier's avatar
Vincent Pelletier committed
730 731
    Create the directory of the partition, assign to the partition user and
    give it the 750 permission. In case if path exists just modifies it.
Łukasz Nowak's avatar
Łukasz Nowak committed
732 733
    """

734
    self.path = os.path.abspath(self.path)
Łukasz Nowak's avatar
Łukasz Nowak committed
735
    owner = self.user if self.user else User('root')
736
    if not os.path.exists(self.path):
Marco Mariani's avatar
Marco Mariani committed
737
      os.mkdir(self.path, 0o750)
Łukasz Nowak's avatar
Łukasz Nowak committed
738
    if alter_user:
Łukasz Nowak's avatar
Łukasz Nowak committed
739
      owner_pw = pwd.getpwnam(owner.name)
740
      os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
741
    os.chmod(self.path, 0o750)
Łukasz Nowak's avatar
Łukasz Nowak committed
742

743 744 745 746 747 748 749 750 751 752 753 754 755 756 757
  def createExternalPath(self, alter_user=True):
    """
    Create and external directory of the partition, assign to the partition user
    and give it the 750 permission. In case if path exists just modifies it.
    """

    for path in self.external_storage_list:
      storage_path = os.path.abspath(path)
      owner = self.user if self.user else User('root')
      if not os.path.exists(storage_path):
        os.mkdir(storage_path, 0o750)
      if alter_user:
        owner_pw = pwd.getpwnam(owner.name)
        os.chown(storage_path, owner_pw.pw_uid, owner_pw.pw_gid)
      os.chmod(storage_path, 0o750)
758

759 760 761 762
  def dump(self):
    """Dump available resources into ~partition_home/.slapos-resource."""
    file_path = os.path.join(self.path, self.resource_file)
    data = _getDict(self)
763 764 765 766 767 768 769
    content = json.dumps(data, sort_keys=True, indent=4)
    if os.path.exists(file_path):
      with open(file_path, "r") as f:
        if f.read() == content:
          # dumped resources didn't change
          return

770
    with open(file_path, "wb") as fo:
771
      fo.write(content)
772
    owner_pw = pwd.getpwnam(self.user.name)
773
    os.chmod(file_path, 0o644)
774 775
    logger.info("Partition resources saved to {}".format(
      self.reference, file_path))
776 777


778
class User(object):
Marco Mariani's avatar
Marco Mariani committed
779 780
  """User: represent and manipulate a user on the system."""

781
  path = None
Łukasz Nowak's avatar
Łukasz Nowak committed
782 783 784 785 786 787 788

  def __init__(self, user_name, additional_group_list=None):
    """
    Attributes:
        user_name: string, the name of the user, who will have is home in
    """
    self.name = str(user_name)
789
    self.shell = '/bin/sh'
Łukasz Nowak's avatar
Łukasz Nowak committed
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
    self.additional_group_list = additional_group_list

  def __getinitargs__(self):
    return (self.name,)

  def setPath(self, path):
    self.path = path

  def create(self):
    """
    Create a user on the system who will be named after the self.name with its
    own group and directory.

    Returns:
        True: if the user creation went right
    """
    # XXX: This method shall be no-op in case if all is correctly setup
    #      This method shall check if all is correctly done
    #      This method shall not reset groups, just add them
Jondy Zhao's avatar
Jondy Zhao committed
809
    grpname = 'grp_' + self.name if sys.platform == 'cygwin' else self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
810
    try:
Jondy Zhao's avatar
Jondy Zhao committed
811
      grp.getgrnam(grpname)
Łukasz Nowak's avatar
Łukasz Nowak committed
812
    except KeyError:
Jondy Zhao's avatar
Jondy Zhao committed
813
      callAndRead(['groupadd', grpname])
Łukasz Nowak's avatar
Łukasz Nowak committed
814

815
    user_parameter_list = ['-d', self.path, '-g', self.name, '-s', self.shell]
Łukasz Nowak's avatar
Łukasz Nowak committed
816 817 818 819
    if self.additional_group_list is not None:
      user_parameter_list.extend(['-G', ','.join(self.additional_group_list)])
    user_parameter_list.append(self.name)
    try:
Łukasz Nowak's avatar
Łukasz Nowak committed
820
      pwd.getpwnam(self.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
821
    except KeyError:
822
      user_parameter_list.append('-r')
Łukasz Nowak's avatar
Łukasz Nowak committed
823 824
      callAndRead(['useradd'] + user_parameter_list)
    else:
825 826
      # if the user is already created and used we should not fail
      callAndRead(['usermod'] + user_parameter_list, raise_on_error=False)
827 828
    # lock the password of user
    callAndRead(['passwd', '-l', self.name])
Łukasz Nowak's avatar
Łukasz Nowak committed
829 830 831 832 833 834 835 836 837 838 839 840 841

    return True

  def isAvailable(self):
    """
    Determine the availability of a user on the system

    Return:
        True: if available
        False: otherwise
    """

    try:
Łukasz Nowak's avatar
Łukasz Nowak committed
842
      pwd.getpwnam(self.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
843 844 845 846
      return True
    except KeyError:
      return False

847

848
class Tap(object):
Łukasz Nowak's avatar
Łukasz Nowak committed
849
  "Tap represent a tap interface on the system"
850 851 852
  IFF_TAP = 0x0002
  TUNSETIFF = 0x400454ca
  KEEP_TAP_ATTACHED_EVENT = threading.Event()
853
  MODE = "tap"
Łukasz Nowak's avatar
Łukasz Nowak committed
854 855 856 857 858

  def __init__(self, tap_name):
    """
    Attributes:
        tap_name: String, the name of the tap interface.
859 860
        ipv4_address: String, local ipv4 to route to this tap
        ipv4_network: String, netmask to use when configure route for this tap
861
        ipv4_gateway: String, ipv4 of gateway to be used to reach local network
Łukasz Nowak's avatar
Łukasz Nowak committed
862 863 864
    """

    self.name = str(tap_name)
865 866 867 868
    self.ipv4_addr = ""
    self.ipv4_netmask = ""
    self.ipv4_gateway = ""
    self.ipv4_network = ""
869 870 871 872 873
    self.ipv6_addr = ""
    self.ipv6_netmask = ""
    self.ipv6_gateway = ""
    self.ipv6_network = ""

Łukasz Nowak's avatar
Łukasz Nowak committed
874 875 876 877

  def __getinitargs__(self):
    return (self.name,)

878
  def createWithOwner(self, owner):
Łukasz Nowak's avatar
Łukasz Nowak committed
879
    """
880
    Create a tap interface on the system if it doesn't exist yet.
Łukasz Nowak's avatar
Łukasz Nowak committed
881 882 883 884
    """
    check_file = '/sys/devices/virtual/net/%s/owner' % self.name
    owner_id = None
    if os.path.exists(check_file):
885 886
      with open(check_file) as fx:
        owner_id = fx.read().strip()
Łukasz Nowak's avatar
Łukasz Nowak committed
887
      try:
888 889
        owner_id = int(owner_id)
      except ValueError:
890 891 892 893 894 895 896 897 898 899 900
        owner_id = pwd.getpwnam(owner_id).pw_uid
      #
      if owner_id != pwd.getpwnam(owner.name).pw_uid:
        logger.warning("Wrong owner of TUN/TAP interface {}! Not touching it."
                       "Expected {:d} got {:d}".format(
          self.name, pwd.getpwnam(owner.name).pw_uid, owner_id))
      # if the interface already exists - don't do anything
      return

    callAndRead(['ip', 'tuntap', 'add', 'dev', self.name, 'mode',
                 self.MODE, 'user', owner.name])
Łukasz Nowak's avatar
Łukasz Nowak committed
901 902
    callAndRead(['ip', 'link', 'set', self.name, 'up'])

903
  def createRoutes(self):
904
    """
905
    Configure ipv4 and ipv6 routes
906
    """
907 908
    if self.ipv4_addr:
      # Check if this route exits
909 910
      code, result = callAndRead(['ip', 'route', 'show', self.ipv4_addr],
                                 raise_on_error=False)
911 912
      if code != 0 or self.ipv4_addr not in result or self.name not in result:
        callAndRead(['ip', 'route', 'add', self.ipv4_addr, 'dev', self.name])
913 914 915
    else:
      raise ValueError("%s should not be empty. No ipv4 address assigned to %s" %
                         (self.ipv4_addr, self.name))
916

917 918
    if self.ipv6_network:
      # Check if this route exits
919 920 921 922 923
      code, result = callAndRead(['ip', '-6', 'route', 'show', self.ipv6_gateway],
                                 raise_on_error=False)
      if code != 0 or self.name not in result:
        callAndRead(['ip', '-6', 'route', 'add', self.ipv6_gateway, 'dev', self.name])

924 925
      code, result = callAndRead(['ip', '-6', 'route', 'show', self.ipv6_network],
                                 raise_on_error=False)
926 927 928 929 930
      if code != 0 or 'via {}'.format(self.ipv6_gateway) not in result or 'dev {}'.format(self.name) not in result:
        if 'dev {}'.format(self.name) in result:
          callAndRead(['ip', '-6', 'route', 'del', self.ipv6_network, 'dev', self.name]) # remove old route without the "via" option
        callAndRead(['ip', '-6', 'route', 'add', self.ipv6_network, 'dev', self.name, 'via', self.ipv6_gateway])

931

932 933 934

class Tun(Tap):
  """Represent TUN interface which might be many per user."""
935

936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
  MODE = "tun"
  BASE_MASK = 12
  BASE_NETWORK = "172.16.0.0"

  def __init__(self, name, sequence=None, partitions=None):
    """Create TUN interface with subnet according to the optional ``sequence`` number.

    :param name: name which will appear in ``ip list`` afterwards
    :param sequence: {int} position of this TUN among all ``partitions``
    """
    super(Tun, self).__init__(name)
    if sequence is not None:
      assert 0 <= sequence < partitions, "0 <= {} < {}".format(sequence, partitions)
      # create base IPNetwork
      ip_network = netaddr.IPNetwork(Tun.BASE_NETWORK + "/" + str(Tun.BASE_MASK))
      # compute shift in BITS to separate ``partitions`` networks into subset
      # example: for 30 partitions we need log2(30) = 8 BITS
      mask_shift = int(math.ceil(math.log(int(partitions), 2.0)))
      # IPNetwork.subnet returns iterator over all possible subnets of given mask
      ip_subnets = list(ip_network.subnet(Tun.BASE_MASK + mask_shift))
      subnet = ip_subnets[sequence]
      # For serialization purposes, convert directly to ``str``
      self.ipv4_network = str(subnet)
      self.ipv4_addr = str(subnet.ip)
      self.ipv4_netmask = str(subnet.netmask)

  def createRoutes(self):
    """Extend for physical addition of network address because TAP let this on external class."""
    if self.ipv4_network:
      # add an address
      code, _ = callAndRead(['ip', 'addr', 'add', self.ipv4_network, 'dev', self.name],
                            raise_on_error=False)
      if code == 0:
        # address added to the interface - wait
        time.sleep(1)
    else:
      raise RuntimeError("Cannot setup address on interface {}. "
                         "Address is missing.".format(self.name))
    # create routes
    super(Tun, self).createRoutes()
    # add iptables rule to accept connections from this interface
    chain_rule = ['INPUT', '-i', self.name, '-j', 'ACCEPT']
    code, _ = callAndRead(['iptables', '-C'] + chain_rule, raise_on_error=False)
    if code == 0:
      # 0 means the rule does not exits so we are free to insert it
      callAndRead(['iptables', '-I'] + chain_rule)


984
class Interface(object):
Marco Mariani's avatar
Marco Mariani committed
985
  """Represent a network interface on the system"""
Łukasz Nowak's avatar
Łukasz Nowak committed
986

987
  def __init__(self, logger, name, ipv4_local_network, ipv6_interface=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
988 989
    """
    Attributes:
990
        name: String, the name of the interface
Łukasz Nowak's avatar
Łukasz Nowak committed
991 992
    """

993
    self._logger = logger
Łukasz Nowak's avatar
Łukasz Nowak committed
994 995
    self.name = str(name)
    self.ipv4_local_network = ipv4_local_network
996
    self.ipv6_interface = ipv6_interface
Łukasz Nowak's avatar
Łukasz Nowak committed
997

998
  # XXX no __getinitargs__, as instances of this class are never deserialized.
Łukasz Nowak's avatar
Łukasz Nowak committed
999 1000

  def getIPv4LocalAddressList(self):
Vincent Pelletier's avatar
Vincent Pelletier committed
1001 1002 1003 1004
    """
    Returns currently configured local IPv4 addresses which are in
    ipv4_local_network
    """
Łukasz Nowak's avatar
Łukasz Nowak committed
1005 1006
    if not socket.AF_INET in netifaces.ifaddresses(self.name):
      return []
Marco Mariani's avatar
Marco Mariani committed
1007 1008 1009 1010 1011 1012 1013 1014 1015
    return [
            {
                'addr': q['addr'],
                'netmask': q['netmask']
                }
            for q in netifaces.ifaddresses(self.name)[socket.AF_INET]
            if netaddr.IPAddress(q['addr'], 4) in netaddr.glob_to_iprange(
                netaddr.cidr_to_glob(self.ipv4_local_network))
            ]
Łukasz Nowak's avatar
Łukasz Nowak committed
1016

1017
  def getGlobalScopeAddressList(self, tap=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
1018
    """Returns currently configured global scope IPv6 addresses"""
1019
    interface_name = self.ipv6_interface or self.name
1020
    try:
Marco Mariani's avatar
Marco Mariani committed
1021
      address_list = [
1022 1023 1024 1025
          q
          for q in netifaces.ifaddresses(interface_name)[socket.AF_INET6]
          if isGlobalScopeAddress(q['addr'].split('%')[0])
      ]
1026
    except KeyError:
1027
      raise ValueError("%s must have at least one IPv6 address assigned" %
1028
                         interface_name)
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039

    if tap:
      try:
        address_list += [
          q
          for q in netifaces.ifaddresses(tap.name)[socket.AF_INET6]
          if isGlobalScopeAddress(q['addr'].split('%')[0])
      ]
      except KeyError:
        pass

Jondy Zhao's avatar
Jondy Zhao committed
1040 1041
    if sys.platform == 'cygwin':
      for q in address_list:
1042
        q.setdefault('netmask', 'FFFF:FFFF:FFFF:FFFF::')
Łukasz Nowak's avatar
Łukasz Nowak committed
1043 1044 1045 1046 1047 1048 1049 1050 1051
    # XXX: Missing implementation of Unique Local IPv6 Unicast Addresses as
    # defined in http://www.rfc-editor.org/rfc/rfc4193.txt
    # XXX: XXX: XXX: IT IS DISALLOWED TO IMPLEMENT link-local addresses as
    # Linux and BSD are possibly wrongly implementing it -- it is "too local"
    # it is impossible to listen or access it on same node
    # XXX: IT IS DISALLOWED to implement ad hoc solution like inventing node
    # local addresses or anything which does not exists in RFC!
    return address_list

1052
  def _addSystemAddress(self, address, netmask, ipv6=True, tap=None):
1053
    """Adds system address to interface
1054

Łukasz Nowak's avatar
Łukasz Nowak committed
1055 1056 1057 1058
    Returns True if address was added successfully.

    Returns False if there was issue.
    """
1059

Łukasz Nowak's avatar
Łukasz Nowak committed
1060
    if ipv6:
1061
      address_string = '%s/%s' % (address, lenNetmaskIpv6(netmask))
Łukasz Nowak's avatar
Łukasz Nowak committed
1062
      af = socket.AF_INET6
1063
      interface_name = self.ipv6_interface or self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
1064 1065 1066
    else:
      af = socket.AF_INET
      address_string = '%s/%s' % (address, netmaskToPrefixIPv4(netmask))
1067
      interface_name = self.name
Łukasz Nowak's avatar
Łukasz Nowak committed
1068

1069 1070 1071

    if tap:
      interface_name = tap.name
Łukasz Nowak's avatar
Łukasz Nowak committed
1072 1073
    # check if address is already took by any other interface
    for interface in netifaces.interfaces():
1074
      if interface != interface_name:
Łukasz Nowak's avatar
Łukasz Nowak committed
1075 1076
        address_dict = netifaces.ifaddresses(interface)
        if af in address_dict:
1077
          if address in [q['addr'].split('%')[0] for q in address_dict[af]]:
1078 1079
            self._logger.warning('Cannot add address %s to %s as it already exists on interface %s.' % \
                (address, interface_name, interface))
Łukasz Nowak's avatar
Łukasz Nowak committed
1080 1081
            return False

Vincent Pelletier's avatar
Vincent Pelletier committed
1082 1083
    if not af in netifaces.ifaddresses(interface_name) \
        or not address in [q['addr'].split('%')[0]
Marco Mariani's avatar
Marco Mariani committed
1084 1085
                           for q in netifaces.ifaddresses(interface_name)[af]
                           ]:
Łukasz Nowak's avatar
Łukasz Nowak committed
1086
      # add an address
1087
      callAndRead(['ip', 'addr', 'add', address_string, 'dev', interface_name])
1088 1089 1090 1091 1092

      # Fake success for local ipv4
      if not ipv6:
        return True

Łukasz Nowak's avatar
Łukasz Nowak committed
1093 1094
      # wait few moments
      time.sleep(2)
1095 1096 1097 1098 1099 1100

    # Fake success for local ipv4
    if not ipv6:
      return True

    # check existence on interface for ipv6
1101
    _, result = callAndRead(['ip', '-6', 'addr', 'list', interface_name])
Łukasz Nowak's avatar
Łukasz Nowak committed
1102 1103
    for l in result.split('\n'):
      if address in l:
1104
        if 'tentative' in l and not tap:
Łukasz Nowak's avatar
Łukasz Nowak committed
1105
          # duplicate, remove
Marco Mariani's avatar
Marco Mariani committed
1106
          callAndRead(['ip', 'addr', 'del', address_string, 'dev', interface_name])
Łukasz Nowak's avatar
Łukasz Nowak committed
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
          return False
        # found and clean
        return True
    # even when added not found, this is bad...
    return False

  def _generateRandomIPv4Address(self, netmask):
    # no addresses found, generate new one
    # Try 10 times to add address, raise in case if not possible
    try_num = 10
    while try_num > 0:
      addr = random.choice([q for q in netaddr.glob_to_iprange(
        netaddr.cidr_to_glob(self.ipv4_local_network))]).format()
1120 1121
      if (dict(addr=addr, netmask=netmask) not in
            self.getIPv4LocalAddressList()):
Łukasz Nowak's avatar
Łukasz Nowak committed
1122 1123 1124 1125 1126 1127 1128 1129 1130
        # Checking the validity of the IPv6 address
        if self._addSystemAddress(addr, netmask, False):
          return dict(addr=addr, netmask=netmask)
        try_num -= 1

    raise AddressGenerationError(addr)

  def addIPv4LocalAddress(self, addr=None):
    """Adds local IPv4 address in ipv4_local_network"""
1131
    netmask = str(netaddr.IPNetwork(self.ipv4_local_network).netmask) if sys.platform == 'cygwin' \
1132
             else '255.255.255.255'
Łukasz Nowak's avatar
Łukasz Nowak committed
1133 1134 1135 1136
    local_address_list = self.getIPv4LocalAddressList()
    if addr is None:
      return self._generateRandomIPv4Address(netmask)
    elif dict(addr=addr, netmask=netmask) not in local_address_list:
1137 1138 1139
      if self._addSystemAddress(addr, netmask, False):
        return dict(addr=addr, netmask=netmask)
      else:
1140
        self._logger.warning('Impossible to add old local IPv4 %s. Generating '
1141
            'new IPv4 address.' % addr)
1142
        return self._generateRandomIPv4Address(netmask)
Łukasz Nowak's avatar
Łukasz Nowak committed
1143 1144 1145 1146
    else:
      # confirmed to be configured
      return dict(addr=addr, netmask=netmask)

1147
  def addIPv6Address(self, addr=None, netmask=None, tap=None):
Łukasz Nowak's avatar
Łukasz Nowak committed
1148
    """
1149
    Adds IPv6 address to interface.
Łukasz Nowak's avatar
Łukasz Nowak committed
1150

1151
    If addr is specified and exists already on interface, do nothing.
Łukasz Nowak's avatar
Łukasz Nowak committed
1152

1153 1154
    If addr is specified and does not exist on interface, try to add given
    address. If it is not possible (ex. because network changed), calculate new
Vincent Pelletier's avatar
Vincent Pelletier committed
1155
    address.
Łukasz Nowak's avatar
Łukasz Nowak committed
1156 1157

    Args:
1158
      addr: Wished address to be added to interface.
Łukasz Nowak's avatar
Łukasz Nowak committed
1159
      netmask: Wished netmask to be used.
1160
      tap: tap interface
Łukasz Nowak's avatar
Łukasz Nowak committed
1161 1162

    Returns:
1163
      dict(addr=address, netmask=netmask).
Łukasz Nowak's avatar
Łukasz Nowak committed
1164 1165 1166

    Raises:
      AddressGenerationError: Couldn't construct valid address with existing
1167 1168
          one's on the interface.
      NoAddressOnInterface: There's no address on the interface to construct
Łukasz Nowak's avatar
Łukasz Nowak committed
1169 1170
          an address with.
    """
1171 1172
    interface_name = self.ipv6_interface or self.name

1173 1174
    # Getting one address of the interface as base of the next addresses
    interface_addr_list = self.getGlobalScopeAddressList()
Łukasz Nowak's avatar
Łukasz Nowak committed
1175
    # No address found
1176 1177 1178
    if len(interface_addr_list) == 0:
      raise NoAddressOnInterface(interface_name)
    address_dict = interface_addr_list[0]
Łukasz Nowak's avatar
Łukasz Nowak committed
1179 1180

    if addr is not None:
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
      # support netifaces which returns netmask and netmask/len
      # will result with updating the computer XML netmask only
      # to follow the change of netmask representation in netifaces
      if '/' in netmask:
        dict_addr_netmask_with_len = dict(addr=addr, netmask=netmask)
        dict_addr_netmask_without_len = dict(addr=addr, netmask=netmask.split('/')[0])
      else:
        dict_addr_netmask_without_len = dict(addr=addr, netmask=netmask)
        dict_addr_netmask_with_len = dict(addr=addr, netmask='%s/%s' % (netmask, lenNetmaskIpv6(netmask)))
      for dict_addr_netmask in [dict_addr_netmask_with_len, dict_addr_netmask_without_len]:
        if dict_addr_netmask in interface_addr_list or \
           (tap and dict_addr_netmask in self.getGlobalScopeAddressList(tap=tap)):
          # confirmed to be configured
          return dict_addr_netmask
1195
      if netmask == address_dict['netmask'] or \
1196
         (tap and lenNetmaskIpv6(netmask) == 128):
Łukasz Nowak's avatar
Łukasz Nowak committed
1197
        # same netmask, so there is a chance to add good one
1198
        interface_network = netaddr.ip.IPNetwork('%s/%s' % (address_dict['addr'],
1199
          lenNetmaskIpv6(address_dict['netmask'])))
Vincent Pelletier's avatar
Vincent Pelletier committed
1200
        requested_network = netaddr.ip.IPNetwork('%s/%s' % (addr,
1201
          lenNetmaskIpv6(address_dict['netmask'])))
1202
        if interface_network.network == requested_network.network:
Łukasz Nowak's avatar
Łukasz Nowak committed
1203
          # same network, try to add
1204
          if self._addSystemAddress(addr, netmask, tap=tap):
Łukasz Nowak's avatar
Łukasz Nowak committed
1205
            # succeed, return it
1206 1207
            self._logger.info('Successfully added IPv6 %s to %s.' % (addr, tap.name or interface_name))
            return dict_addr_netmask
1208
          else:
1209
            self._logger.warning('Impossible to add old public IPv6 %s. '
1210
                'Generating new IPv6 address.' % addr)
Łukasz Nowak's avatar
Łukasz Nowak committed
1211 1212 1213 1214

    # Try 10 times to add address, raise in case if not possible
    try_num = 10
    netmask = address_dict['netmask']
1215
    if tap:
1216
      netmask_len = lenNetmaskIpv6(netmask)
1217 1218 1219 1220 1221 1222
      prefix = binFromIpv6(address_dict['addr'])[:netmask_len]
      netmask_len += 16
      # we generate a subnetwork for the tap
      # the subnetwork has 16 bits more than the interface network
      # make sure we have at least 2 IPs in the subnetwork
      if netmask_len >= 128:
1223
        self._logger.error('Interface %s has netmask %s which is too big for generating IPv6 on taps.' % (interface_name, netmask))
1224
        raise AddressGenerationError(addr)
1225
      netmask = ipv6FromBin('1'*128) # the netmask of the tap itself is always 128 bits
1226

Łukasz Nowak's avatar
Łukasz Nowak committed
1227
    while try_num > 0:
1228
      if tap: