merge master into slave_instance branch

buildout.cfg

@@ -3,6 +3,7 @@ develop = .
 parts =
   slapos
   pyflakes
+  test
 
 find-links =
   http://www.nexedi.org/static/packages/source/slapos.buildout/
@@ -44,5 +45,10 @@ eggs =
   rstctl
 interpreter = python
 
+[test]
+recipe = zc.recipe.testrunner
+eggs =
+  slapos.core
+
 [versions]
 zc.buildout = 1.5.3-dev-SlapOS-005

master/bt5/vifib_base/DocumentTemplateItem/SoftwareInstance.py

@@ -29,6 +29,13 @@ from AccessControl import ClassSecurityInfo
 from Products.ERP5Type import Permissions
 from Products.ERP5.Document.Item import Item
 from lxml import etree
+import collections
+
+class DisconnectedSoftwareTree(Exception):
+  pass
+
+class CyclicSoftwareTree(Exception):
+  pass
 
 class SoftwareInstance(Item):
   """
@@ -60,3 +67,44 @@ class SoftwareInstance(Item):
           value = element.text
         result_dict[key] = value
     return result_dict
+
+  security.declareProtected(Permissions.AccessContentsInformation,
+    'checkNotCyclic')
+  def checkNotCyclic(self, graph):
+    # see http://neopythonic.blogspot.com/2009/01/detecting-cycles-in-directed-graph.html
+    todo = set(graph.keys())
+    while todo:
+      node = todo.pop()
+      stack = [node]
+      while stack:
+        top = stack[-1]
+        for node in graph[top]:
+          if node in stack:
+            raise CyclicSoftwareTree
+          if node in todo:
+            stack.append(node)
+            todo.remove(node)
+            break
+        else:
+          node = stack.pop()
+    return True
+
+  security.declareProtected(Permissions.AccessContentsInformation,
+    'checkConnected')
+  def checkConnected(self, graph, root):
+    size = len(graph)
+    visited = set()
+    to_crawl = collections.deque(graph[root])
+    while to_crawl:
+      current = to_crawl.popleft()
+      if current in visited:
+        continue
+      visited.add(current)
+      node_children = set(graph[current])
+      to_crawl.extend(node_children - visited)
+    # add one to visited, as root won't be visited, only children
+    # this is false positive in case of cyclic graphs, but they are
+    # anyway wrong in Software Instance trees
+    if size != len(visited) + 1:
+      raise DisconnectedSoftwareTree
+    return True

master/bt5/vifib_base/WorkflowTemplateItem/portal_workflow/software_instance_slap_interface_workflow/scripts/SoftwareInstance_requestSoftwareInstance.xml

@@ -69,9 +69,12 @@ if is_slave == True:\n
 else:\n
   software_instance_portal_type = "Software Instance"\n
 \n
-# Get root software instance\n
+# graph allows to "simulate" tree change after requested operation\n
+graph = {}\n
+# Get root software instance and create initial graph\n
 predecessor_software_instance = software_instance\n
 while (predecessor_software_instance is not None):\n
+  graph[predecessor_software_instance.getUid()] = predecessor_software_instance.getPredecessorUidList()\n
   root_software_instance = predecessor_software_instance\n
   predecessor_software_instance = predecessor_software_instance.getPredecessorRelatedValue(\n
       portal_type="Software Instance")\n
@@ -89,6 +92,11 @@ request_software_instance = software_instance.portal_catalog.getResultValue(\n
   root_uid=root_software_instance.getUid(),\n
 )\n
 \n
+# above query does not find root software instance, but as this case is easy\n
+# to find, just check if such request does not come and raise\n
+if root_software_instance.getTitle() == requested_partition_reference:\n
+  raise ValueError(\'It is disallowed to request root software instance\')\n
+\n
 if (request_software_instance is None):\n
   if (portal.portal_activities.countMessageWithTag(tag) > 0):\n
     # The software instance is already under creation but can not be fetched from catalog\n
@@ -134,6 +142,8 @@ else:\n
     predecessor_software_instance.edit(\n
       predecessor_uid_list=predecessor_uid_list,\n
       activate_kw={\'tag\': tag},)\n
+    graph[predecessor_software_instance.getUid()] = predecessor_uid_list\n
+\n
   if state == \'started\':\n
     request_software_instance.startRequested()\n
     request_software_instance.activate(after_tag=tag).requestStartComputerPartition()\n
@@ -142,8 +152,17 @@ else:\n
     request_software_instance.activate(after_tag=tag).requestStopComputerPartition()\n
   else:\n
     raise ValueError(\'State %r is not supported\' % state)\n
-\n
 predecessor_list = software_instance.getPredecessorList() + [request_software_instance.getRelativeUrl()]\n
+# Add requested software instance to graph if does not exists there yet\n
+if not request_software_instance.getUid() in graph:\n
+  graph[request_software_instance.getUid()] = []\n
+\n
+# update graph to reflect requested operation\n
+graph[software_instance.getUid()] = software_instance.getPredecessorUidList() + [request_software_instance.getUid()]\n
+\n
+# check if all elements are still connected\n
+software_instance.checkNotCyclic(graph)\n
+software_instance.checkConnected(graph, root_software_instance.getUid())\n
 \n
 software_instance.edit(\n
   predecessor_list=predecessor_list,\n

master/bt5/vifib_base/bt/revision

-264
\ No newline at end of file
+265

master/bt5/vifib_web/bt/revision

-300
\ No newline at end of file
+301

setup.py

@@ -57,4 +57,5 @@ setup(name=name,
           'slapproxy = slapos.proxy:main',
         ]
       },
+      test_suite="slapos.tests",
     )

slapos/format.py

@@ -27,13 +27,13 @@
 ##############################################################################
 from optparse import OptionParser, Option
 from xml_marshaller import xml_marshaller
-from pwd import getpwnam
 import ConfigParser
 import grp
 import logging
 import netaddr
 import netifaces
 import os
+import pwd
 import random
 import slapos.slap as slap
 import socket
@@ -317,7 +317,7 @@ class Computer:
     slapsoft.path = self.software_root
     if alter_user:
       slapsoft.create()
-      slapsoft_pw = getpwnam(slapsoft.name)
+      slapsoft_pw = pwd.getpwnam(slapsoft.name)
       os.chown(self.software_root, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
     os.chmod(self.software_root, 0755)
 
@@ -415,7 +415,7 @@ class Partition:
     if not os.path.exists(self.path):
       os.mkdir(self.path, 0750)
     if alter_user:
-      owner_pw = getpwnam(owner.name)
+      owner_pw = pwd.getpwnam(owner.name)
       os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
     os.chmod(self.path, 0750)
 
@@ -457,7 +457,7 @@ class User:
       user_parameter_list.extend(['-G', ','.join(self.additional_group_list)])
     user_parameter_list.append(self.name)
     try:
-      getpwnam(self.name)
+      pwd.getpwnam(self.name)
     except KeyError:
       callAndRead(['useradd'] + user_parameter_list)
     else:
@@ -475,7 +475,7 @@ class User:
     """
 
     try:
-      getpwnam(self.name)
+      pwd.getpwnam(self.name)
       return True
 
     except KeyError:
@@ -572,7 +572,7 @@ class Tap:
         owner_id = int(open(check_file).read().strip())
       except Exception:
         pass
-    if (owner_id is None) or (owner_id != getpwnam(owner.name).pw_uid):
+    if (owner_id is None) or (owner_id != pwd.getpwnam(owner.name).pw_uid):
       callAndRead(['tunctl', '-t', self.name, '-u', owner.name])
     callAndRead(['ip', 'link', 'set', self.name, 'up'])
 
@@ -584,7 +584,7 @@ class Tap:
 class Bridge:
   "Bridge represent a bridge on the system"
 
-  def __init__(self, name, ipv4_local_network, ipv6_interface):
+  def __init__(self, name, ipv4_local_network, ipv6_interface=None):
     """
     Attributes:
         name: String, the name of the bridge
@@ -846,11 +846,14 @@ class Parser(OptionParser):
         help="Shall slapformat alter network configuration [default: True]"),
       ])
 
-  def check_args(self):
+  def check_args(self, args):
     """
     Check arguments
     """
-    (options, args) = self.parse_args()
+    if args:
+      (options, args) = self.parse_args(list(args))
+    else:
+      (options, args) = self.parse_args()
     if len(args) != 1:
       self.error("Incorrect number of arguments")
     return options, args[0]
@@ -1093,17 +1096,17 @@ class Config:
     self.computer_xml = os.path.abspath(self.computer_xml)
 
 
-def main():
+def main(*args):
   "Run default configuration."
   global os
   global callAndRead
-  global getpwnam
+  global pwd
   real_callAndRead = callAndRead
   usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]
 
   try:
     # Parse arguments
-    options, configuration_file_path = Parser(usage=usage).check_args()
+    options, configuration_file_path = Parser(usage=usage).check_args(args)
     config = Config()
     config.setConfig(options, configuration_file_path)
     os = OS(config)
@@ -1125,7 +1128,7 @@ def main():
           pw_uid = 12345
           pw_gid = 54321
         return result
-      getpwnam = fake_getpwnam
+      pwd.getpwnam = fake_getpwnam
     else:
       dry_callAndRead = real_callAndRead
     if config.verbose:

slapos/grid/slapgrid.py

@@ -337,7 +337,7 @@ class Slapgrid(object):
       computer_partition_list = self.computer.getComputerPartitionList()
     except socket.error as error:
       self.logger.fatal(error)
-      sys.exit(1)
+      raise
     return computer_partition_list
 
   def processSoftwareReleaseList(self):

slapos/tests/mock/httplib.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Mocked httplib
+"""
+
+from urlparse import urlparse
+
+__all__ = []
+
+def log(message):
+  """Need to be overridden to get a proper logger
+  """
+  pass
+
+class HTTPConnection(object):
+
+  scheme = 'http'
+
+  def _callback(self, path, method, body, headers):
+    """To get it works properly, you need to override
+    HTTPConnection._callback.  This method received the instance, the path,
+    method and request body as parameter, and it has to return a tuple with
+    headers dictionary and body response string.
+    
+    @param self object instance reference
+    @param URL the parsed URL
+    @param method the http method
+    @param body the request body
+    @param headers the request headers
+    
+    @return tuple containing status integer, headers dictionary and body
+    response"""
+    return (0, {}, '', )
+
+  def __init__(self, host, port=None, strict=None,
+               timeout=None, source_address=None):
+    self.host = host
+    self.port = port
+    self.strict = strict
+    self.timeout = timeout
+    self.source_address = source_address
+    self.__response = None
+
+  def request(self, method, url, body=None, headers=None):
+    status, headers, body = self._callback(url, method, body, headers)
+    self.__response = HTTPResponse('HTTP/1.1', 200, 'OK', body, headers)
+
+
+  def getresponse(self):
+    response = self.__response
+    self.__response = None
+    return response
+
+  def set_debuglevel(self, level):
+    pass
+
+  def set_tunnel(self, host, port=None, headers=None):
+    pass
+
+  def connect(self):
+    pass
+
+  def close(self):
+    pass
+
+  def putrequest(self, request, selector, skip_host=None,
+                 skip_accept_encoding=None):
+    pass
+
+  def putheader(self, *args):
+    pass
+
+  def endheaders(self):
+    pass
+
+  def send(self, data):
+    pass
+
+class HTTPSConnection(HTTPConnection):
+
+  def __init__(self, host, port=None, key_file=None,
+               cert_file=None, strict=None, timeout=None,
+               source_address=None):
+    super().__init__(self, host, port, strict, timeout,
+                     source_address)
+    pass
+
+class HTTPResponse(object):
+
+  def __init__(self, version, status, reason, content, headers=()):
+    self.version = version
+    self.status = status
+    self.reason = reason
+    self.__headers = headers
+    self.__content = content
+
+  def read(self, amt=None):
+    result = None
+    if amt is None:
+      result = self.__content
+      self.__content = ''
+    else:
+      end = max(amt, len(self.__content))
+      result = self.__content[:end]
+      del self.__content[:end]
+    return result
+
+  def getheader(self, name, default=None):
+    pass
+
+  def getheaders(self):
+    pass