testnode: Modify testnode structure to include scalability tests.

Testnode code has been splitted. Runner class has been created for
each test types (unit and scalability).
Communication with SlapOS and ERP5 Master has been added.

erp5/util/taskdistribution/__init__.py

@@ -138,6 +138,15 @@ class TestResultLineProxy(RPCRetry):
     def name(self):
         return self._name
 
+    def isTestCaseAlive(self):
+        """
+        Tell if test result line is still alive on site.
+        """
+        try:
+          return bool(self._retryRPC('isTestCaseAlive', [self._test_result_line_path]))
+        except:
+          raise ValueError('isTestCaseAlive Failed.')
+
     def stop(self, test_count=None, error_count=None, failure_count=None,
             skip_count=None, duration=None, date=None, command=None,
             stdout=None, stderr=None, html_test_result=None, **kw):
@@ -201,6 +210,10 @@ class TestResultProxy(RPCRetry):
         return '<%s(%r, %r, %r) at %x>' % (self.__class__.__name__,
             self._test_result_path, self._node_title, self._revision, id(self))
 
+    @property
+    def test_result_path(self):
+        return self._test_result_path
+
     @property
     def revision(self):
         return self._revision
@@ -350,6 +363,35 @@ class TestResultProxy(RPCRetry):
         if self._watcher_thread is not None:
           self._watcher_thread.join()
 
+    def stop(self):
+        """
+        
+        """
+        return self._retryRPC('stopTest', [self._test_result_path])
+        
+class TestResultProxyProxy(TestResultProxy):
+    """
+    A wrapper/proxy to TestResultProxy
+    """
+    def __init__(self, test_suite_master_url, retry_time, logger, test_result_path,
+                node_title, revision):
+      try:
+        proxy = ServerProxy(
+                test_suite_master_url,
+                allow_none=True,
+            ).portal_task_distribution
+      except:
+        raise ValueError("Cannot instanciate ServerProxy")
+      TestResultProxy.__init__(self, proxy, retry_time, logger, test_result_path,
+                node_title, revision)
+
+    def getRunningTestCase(self):
+      """
+      A proxy to getNextTestCase
+      Return the relative path of the test with the running state
+      """
+      return self._retryRPC('getRunningTestCase', [self._test_result_path])
+
 class ServerProxy(xmlrpclib.ServerProxy):
 
     def __init__(self, *args, **kw):
@@ -427,7 +469,6 @@ class TaskDistributionTool(RPCRetry):
 class TaskDistributor(RPCRetry):
 
     def __init__(self,portal_url,retry_time=64,logger=None):
-
         if logger is None:
            logger = null_logger
         if portal_url is None:
@@ -440,14 +481,67 @@ class TaskDistributor(RPCRetry):
             raise ValueError('Unsupported protocol revision: %r',
                 protocol_revision)
 
-    def startTestSuite(self,node_title):
+    def startTestSuite(self,node_title,computer_guid='unknown'):
       """
         Returns None if no test suite is needed.
         therwise, returns a JSON with all the test suite parameters.
       """
-      result = self._retryRPC('startTestSuite',(node_title,))
+      result = self._retryRPC('startTestSuite',(node_title,computer_guid,))
       return result
 
+    def getTestType(self):
+      """
+        Return the Test Type
+      """
+      result = self._retryRPC('getTestType')
+      return result
+
+    def subscribeNode(self, node_title, computer_guid):
+      """
+        Susbscribes node with the node title and the computer guid.
+      """
+      self._retryRPC('subscribeNode', (node_title,computer_guid,))
+
+
+    def generateConfiguration(self, test_suite_title):
+      """
+        Generates a configuration from a test_suite_title
+      """
+      return self._retryRPC('generateConfiguration', (test_suite_title,))
+
+
+    def isMasterTestnode(self, test_node_title):
+      """
+        Returns True or False if the testnode is the master
+      """
+      return self._retryRPC('isMasterTestnode', (test_node_title,))
+
+    def getSlaposAccountKey(self):
+      """
+        Returns the slapos account key related to the distributor
+      """
+      return self._retryRPC('getSlaposAccountKey')
+    
+    def getSlaposAccountCertificate(self):
+      """
+        Returns the slapos account certificate related to the distributor
+      """
+      return self._retryRPC('getSlaposAccountCertificate')
+
+    def getSlaposUrl(self):
+      """
+        Returns the url of slapos master related to the distributor
+      """
+      return self._retryRPC('getSlaposUrl')
+      
+    def getSlaposHateoasUrl(self):
+      """
+        Returns the url of API REST using hateoas of
+        slapos master related to the distributor
+      """
+      return self._retryRPC('getSlaposHateoasUrl')
+
+
 class DummyTaskDistributionTool(object):
     """
     Fake remote server.

erp5/util/testnode/NodeTestSuite.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from datetime import datetime,timedelta
+import os
+import subprocess
+import sys
+import time
+import glob
+import SlapOSControler
+import json
+import time
+import shutil
+import logging
+import string
+import random
+from ProcessManager import SubprocessError, ProcessManager, CancellationError
+from subprocess import CalledProcessError
+from Updater import Updater
+from erp5.util import taskdistribution
+
+class SlapOSInstance(object):
+  """
+  Base of an software instance,
+  store variables used during software installation
+  """
+  def __init__(self):
+    self.retry_software_count = 0
+    self.retry = False
+
+  def edit(self, **kw):
+    self.__dict__.update(**kw)
+    self._checkData()
+
+  def _checkData(self):
+    pass
+    
+    
+class NodeTestSuite(SlapOSInstance):
+  """
+  
+  """
+  def __init__(self, reference):
+    super(NodeTestSuite, self).__init__()
+    self.reference = reference
+
+  def edit(self, **kw):
+    super(NodeTestSuite, self).edit(**kw)
+
+  def _checkData(self):
+    if getattr(self, "working_directory", None) is not None:
+      if not(self.working_directory.endswith(os.path.sep + self.reference)):
+        self.working_directory = os.path.join(self.working_directory,
+                                             self.reference)
+      SlapOSControler.createFolder(self.working_directory)
+      self.test_suite_directory = os.path.join(
+                                   self.working_directory, "test_suite")
+      self.custom_profile_path = os.path.join(self.working_directory,
+                                 'software.cfg')
+    if getattr(self, "vcs_repository_list", None) is not None:
+      for vcs_repository in self.vcs_repository_list:
+        buildout_section_id = vcs_repository.get('buildout_section_id', None)
+        repository_id = buildout_section_id or \
+                        vcs_repository.get('url').split('/')[-1].split('.')[0]
+        repository_path = os.path.join(self.working_directory,repository_id)
+        vcs_repository['repository_id'] = repository_id
+        vcs_repository['repository_path'] = repository_path
+
+  def createSuiteLog(self):
+    # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
+    alphabets = string.digits + string.letters
+    rand_part = ''.join(random.choice(alphabets) for i in xrange(32))
+    random_suite_folder_id = '%s-%s' % (self.reference, rand_part)
+    suite_log_directory = os.path.join(self.log_directory,
+                                       random_suite_folder_id)
+    SlapOSControler.createFolders(suite_log_directory)
+    self.suite_log_path = os.path.join(suite_log_directory,
+                                       'suite.log')
+    return self.getSuiteLogPath(), random_suite_folder_id
+
+  def getSuiteLogPath(self):
+    return getattr(self,"suite_log_path", None)
+

erp5/util/testnode/SlapOSControler.py

@@ -33,6 +33,8 @@ import xml_marshaller
 import shutil
 import sys
 import glob
+import argparse
+from slapos import client
 
 MAX_PARTIONS = 10
 MAX_SR_RETRIES = 3
@@ -47,6 +49,20 @@ def createFolders(folder):
   if not(os.path.exists(folder)):
     os.makedirs(folder)
 
+def isDir(folder):
+  return os.path.isdir(folder)
+
+def createFile(path, mode, content):
+  f = open(path, mode)
+  if os.path.exists(path):
+    f.write(content)
+    f.close()
+  else:
+    # error
+    pass
+
+
+
 class SlapOSControler(object):
 
   def __init__(self, working_directory, config, log):
@@ -54,8 +70,175 @@ class SlapOSControler(object):
     self.software_root = os.path.join(working_directory, 'soft')
     self.instance_root = os.path.join(working_directory, 'inst')
     self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
-    self.proxy_database = os.path.join(working_directory, 'proxy.db')
     self.log = log
+    self.proxy_database = os.path.join(working_directory, 'proxy.db')
+    self.instance_config = {}
+
+  #TODO: implement a method to get all instance related the slapOS account
+  # and deleting all old instances (based on creation date or name etc...)
+
+  def createSlaposConfigurationFileAccount(self, key, certificate, slapos_url, config):
+    # Create "slapos_account" directory in the "slapos_directory"
+    slapos_account_directory = os.path.join(config['slapos_directory'], "slapos_account")
+    createFolder(slapos_account_directory)
+    # Create slapos-account files
+    slapos_account_key_path = os.path.join(slapos_account_directory, "key")
+    slapos_account_certificate_path = os.path.join(slapos_account_directory, "certificate")
+    configuration_file_path = os.path.join(slapos_account_directory, "slapos.cfg")
+    configuration_file_value = "[slapos]\nmaster_url = %s\n\
+[slapconsole]\ncert_file = %s\nkey_file = %s" %(
+                                  slapos_url,
+                                  slapos_account_certificate_path,
+                                  slapos_account_key_path)
+    createFile(slapos_account_key_path, "w", key)
+    createFile(slapos_account_certificate_path, "w", certificate)
+    createFile(configuration_file_path, "w", configuration_file_value)
+    self.configuration_file_path = configuration_file_path
+    return slapos_account_key_path, slapos_account_certificate_path, configuration_file_path
+
+  def supply(self, software_url, computer_id, state="available"):
+    """
+    Request the installation of a software release on a specific node
+    Ex :
+    my_controler.supply('kvm.cfg', 'COMP-726')
+    """
+    self.log('SlapOSControler : supply')
+    parser = argparse.ArgumentParser()
+    parser.add_argument("configuration_file")
+    parser.add_argument("software_url")
+    parser.add_argument("node")
+    if os.path.exists(self.configuration_file_path):
+      args = parser.parse_args([self.configuration_file_path, software_url, computer_id])
+      config = client.Config()
+      config.setConfig(args, args.configuration_file)
+      try:
+        local = client.init(config)
+        local['supply'](software_url, computer_guid=computer_id, state=state)
+        self.log('SlapOSControler : supply %s %s %s' %(software_url, computer_id, state))
+      except:
+        self.log("SlapOSControler.supply, \
+                 exception in registerOpenOrder", exc_info=sys.exc_info())
+        raise ValueError("Unable to supply (or remove)")
+    else:
+      raise ValueError("Configuration file not found.")
+
+  def destroy(self, software_url, computer_id):
+    """
+    Request Deletetion of a software release on a specific node
+    Ex :
+    my_controler.destroy('kvm.cfg', 'COMP-726')
+    """
+    self.supply(self, software_url, computer_id, state="destroyed")
+    
+  def getInstanceRequestedState(self, reference):
+    try:
+      return self.instance_config[reference]['requested_state']
+    except:
+      raise ValueError("Instance '%s' not exist" %self.instance_config[reference])
+      
+  def request(self, reference, software_url, software_type=None,
+            software_configuration=None, computer_guid=None, state='started'):
+    """
+    configuration_file_path (slapos acount)
+    reference : instance title
+    software_url : software path/url
+    software_type : scalability
+    software_configuration : dict { "_" : "{'toto' : 'titi'}" } 
+
+    Ex :
+    my_controler._request('Instance16h34Ben',
+                               'kvm.cfg', 'cluster', { "_" : "{'toto' : 'titi'}" } )
+
+    """
+    self.log('SlapOSControler : request-->SlapOSMaster')
+    current_intance_config = {'software_type':software_type,
+                              'software_configuration':software_configuration,
+                              'computer_guid':computer_guid,
+                              'software_url':software_url,
+                              'requested_state':state,
+                              'partition':None
+                              }
+    self.instance_config[reference] = current_intance_config
+
+    filter_kw = None
+    if computer_guid != None:
+      filter_kw = { "computer_guid": computer_guid }
+    if os.path.exists(self.configuration_file_path):
+      parser = argparse.ArgumentParser()
+      parser.add_argument("configuration_file")
+      args = parser.parse_args([self.configuration_file_path])
+      config = client.Config()
+      config.setConfig(args, args.configuration_file)
+      try:
+        local = client.init(config)
+        partition = local['request'](
+          software_release = software_url,
+          partition_reference = reference,
+          partition_parameter_kw = software_configuration,
+          software_type = software_type,
+          filter_kw = filter_kw,
+          state = state)
+        self.instance_config[reference]['partition'] = partition
+        if state == 'destroyed':
+          del self.instance_config[reference]
+        if state == 'started':
+          self.log('Instance started with configuration: %s' %str(software_configuration))
+      except:
+        self.log("SlapOSControler.request, \
+              exception in registerOpenOrder", exc_info=sys.exc_info())
+        raise ValueError("Unable to do this request")
+    else:
+      raise ValueError("Configuration file not found.")
+
+  def _requestSpecificState(self, reference, state):
+    self.request(reference,
+        self.instance_config[reference]['software_url'],
+        self.instance_config[reference]['software_type'],
+        self.instance_config[reference]['software_configuration'],
+        self.instance_config[reference]['computer_guid'],
+        state=state
+    )    
+  
+  def destroyInstance(self, reference):
+    self.log('SlapOSControler : delete instance')
+    try:
+      self._requestSpecificState(reference, 'destroyed')
+    except:
+      raise ValueError("Can't delete instance '%s' (instance may not been created?)" %reference)
+    
+  def stopInstance(self, reference):
+    self.log('SlapOSControler : stop instance')
+    try:
+      self._requestSpecificState(reference, 'stopped')
+    except:
+      raise ValueError("Can't stop instance '%s' (instance may not been created?)" %reference)
+  
+  def startInstance(self, reference):
+    self.log('SlapOSControler : start instance')
+    try:
+      self._requestSpecificState(reference, 'started')
+    except:
+      raise ValueError("Can't start instance '%s' (instance may not been created?)" %reference)
+
+  def updateInstanceXML(self, reference, software_configuration):
+    """
+    Update the XML configuration of an instance
+    # Request same instance with different parameters.
+    """
+    self.log('SlapOSControler : updateInstanceXML')
+    self.log('SlapOSControler : updateInstanceXML will request same'
+             'instance with new XML configuration...')
+
+    try:
+      self.request(reference,
+        self.instance_config[reference]['software_url'],
+        self.instance_config[reference]['software_type'],
+        software_configuration,
+        self.instance_config[reference]['computer_guid'],
+        state='started'
+      )
+    except:
+      raise ValueError("Can't update instance '%s' (may not exist?)" %reference)
 
   def _resetSoftware(self):
     self.log('SlapOSControler : GOING TO RESET ALL SOFTWARE : %r' %
@@ -65,7 +248,6 @@ class SlapOSControler(object):
     os.mkdir(self.software_root)
     os.chmod(self.software_root, 0750)
 
-
   def initializeSlapOSControler(self, slapproxy_log=None, process_manager=None,
         reset_software=False, software_path_list=None):
     self.process_manager = process_manager

erp5/util/testnode/SlapOSMasterCommunicator.py

+import json
+import httplib
+import urlparse
+import time
+
+TIMEOUT = 30
+
+# TODO: News-> look list to get last news... (and not the first of the list)
+
+class SlapOSMasterCommunicator(object):
+  """
+  Communication with slapos Master using Hateoas.
+  
+  collection: collection of data (hosting_subscription, instance, software_release...)
+  hosting_subscription: result of a request
+  instance(s): instance(s) related to an hosting_subscription
+
+  usage: ex:
+    # Try to reuse same communicator, because initilization step may takes a lot of time
+    # due to listing of all instances (alive or not) related to the specified slapOS account.
+    communicator = SlapOSMasterCommunicator()
+    
+    # Print news related to 'TestScalability_21423104630420' all instances
+    instance_link_list =  communicator._getRelatedInstanceLink('TestScalability_21423104630420')
+    for instance_link in instance_link_list:
+      news = communicator.getNewsFromInstanceLink(instance_link)
+      print news['news']
+  """
+  def __init__(self, certificate_path, key_path, log,
+                     url):
+    # Create connection
+    api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url)
+    self.log = log
+    self.certificate_path = certificate_path
+    self.key_path = key_path
+    self.url = url
+    self.connection = self._getConnection(self.certificate_path, self.key_path, self.url)
+    # Get master
+    master_link = {'href':api_path,'type':"application/vnd.slapos.org.hal+json; class=slapos.org.master"}
+    master = self._curl(master_link)
+    self.person_link = master['_links']['http://slapos.org/reg/me']
+    # Get person related to specified key/certificate provided
+    person = self._curl(self.person_link)
+    self.personnal_collection_link = person['_links']['http://slapos.org/reg/hosting_subscription']
+    # Get collection (of hosting subscriptions)
+    collection = self._curl(self.personnal_collection_link)
+    # XXX: This part may be extremly long (because here no hosting subscriptions
+    # has been visited)
+    self.hosting_subcriptions_dict = {}
+    self.visited_hosting_subcriptions_link_list = []
+    self.log("SlapOSMasterCommunicator will read all hosting subscriptions entries, "
+             "it may take several time...")
+    self._update_hosting_subscription_informations()
+    
+  def _getConnection(self,certificate_path, key_path, url):
+    api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url)
+    #self.log("HTTPS Connection with: %s, cert=%s, key=%s" %(api_netloc,key_path,certificate_path))
+    return httplib.HTTPSConnection(api_netloc, key_file=key_path, cert_file=certificate_path, timeout=TIMEOUT)
+
+  def _curl(self, link):
+    """
+    'link' must look like : {'href':url,'type':content_type}
+    """
+    # Set timeout
+    import socket
+    socket.setdefaulttimeout(1.0*TIMEOUT)
+    
+    api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(link['href'])
+    max_retry = 10
+    # Try to use existing conection
+    try:
+      self.connection.request(method='GET', url=api_path, headers={'Accept': link['type']}, body="")
+      response = self.connection.getresponse()
+      return json.loads(response.read())
+    # Create and use new connection
+    except:
+      retry = 0
+      # (re)Try several time to use new connection
+      while retry < max_retry:
+        try:
+          self.connection = self._getConnection(self.certificate_path, self.key_path, self.url)
+          self.connection.request(method='GET', url=api_path, headers={'Accept': link['type']}, body="")
+          response = self.connection.getresponse()
+          return json.loads(response.read())
+        except:
+          self.log("SlapOSMasterCommunicator: Connection failed..")
+          retry += 1
+          time.sleep(10)
+    self.log("SlapOSMasterCommunicator: All connection attempts failed after %d try.." %max_retry)
+    raise ValueError("SlapOSMasterCommunicator: Impossible to use connection")
+        
+  def _update_hosting_subscription_informations(self):
+    """
+    Add all not already visited hosting_subcription
+    # Visit all hosting subscriptions and fill a dict containing all
+    # new hosting subscriptions. ( like: {hs1_title:hs1_link, hs2_title:hs2_link, ..} )
+    # and a list of visited hosting_subsciption ( like: [hs1_link, hs2_link, ..] )
+    """
+    collection = self._curl(self.personnal_collection_link)
+    # For each hosting_subcription present in the collection
+    for hosting_subscription_link in collection['_links']['item']:
+      if hosting_subscription_link not in self.visited_hosting_subcriptions_link_list:
+        hosting_subscription = self._curl(hosting_subscription_link)
+        self.hosting_subcriptions_dict.update({hosting_subscription['title']:hosting_subscription_link})
+        self.visited_hosting_subcriptions_link_list.append(hosting_subscription_link)
+  
+  def _getRelatedInstanceLink(self, hosting_subscription_title):
+    """
+    Return a list of all related instance_url from an hosting_subscription_title
+    """
+    # Update informations
+    self._update_hosting_subscription_informations()
+    # Get specified hosting_subscription
+    hosting_subscription_link = self.hosting_subcriptions_dict[hosting_subscription_title]
+    hosting_subscription = self._curl(hosting_subscription_link)
+    assert(hosting_subscription_title == hosting_subscription['title'])
+    # Get instance collection related to this hosting_subscription
+    instance_collection_link = hosting_subscription['_links']['http://slapos.org/reg/instance']
+    instance_collection = self._curl(instance_collection_link)
+    related_instance_link_list = []
+    # For each instance present in the collection
+    for instance in instance_collection['_links']['item']:
+      related_instance_link_list.append(instance)
+    return related_instance_link_list
+
+  def getNewsFromInstanceLink(self, instance_link):
+      instance = self._curl(instance_link)
+      news_link = instance['_links']['http://slapos.org/reg/news']
+      return self._curl(news_link)
+
+  def isHostingSubsciptionStatusEqualTo(self, hosting_subscription_title, excepted_news_text):
+    """
+    Return True if all related instance state are equal to status,
+    or False if not or if there is are no related instances.
+    """
+    related_instance_link_list = _getRelatedInstanceLink(hosting_subscription_title)
+    # For each instance present in the collection
+    for instance_link in related_instance_link_list:
+      news = self.getNewsFromInstanceLink(instance_link)
+      if excepted_news_text != news['news'][0]['text']:
+        return False
+    return len(related_instance_link_list) > 0
+
+  def isInstanceReady(self, instance_link, status):
+    """
+    Return True if instance status and instance news text ~looks corresponding.
+    ( use the matching of 'correctly' and 'Instance' and status )
+    """
+    # XXX: SlapOS Master doesn't store any "news" about slave instances. Assume true.
+    if self._curl(instance_link)['slave']:
+      return True
+    text = self.getNewsFromInstanceLink(instance_link)['news'][0]['text']
+    return ('Instance' in text) and ('correctly' in text) and (status in text)
+
+  # check if provided 'status' = status
+  def isHostingSubscriptionReady(self, hosting_subscription_title, status):
+    """
+    Return True if all instance status and instance news text ~looks corresponding.
+    ( use the matching of 'correctly' and 'Instance' and status ).
+    """
+    instance_link_list = self._getRelatedInstanceLink(hosting_subscription_title)
+    for instance_link in instance_link_list:
+      if not self.isInstanceReady(instance_link, status):
+        return False
+    return len(instance_link_list) > 0
+    
+  def isRegisteredHostingSubscription(self, hosting_subscription_title):
+    """
+    Return True if the specified hosting_subscription is present on SlapOSMaster
+    """
+    self._update_hosting_subscription_informations()
+    if self.hosting_subcriptions_dict.get(hosting_subscription_title):
+      return True
+    return False
+
+  def getHostingSubscriptionDict(self):
+    """
+    Return the dict of hosting subcription.
+    """
+    return self.hosting_subcriptions_dict
+
+  def getHostingSubscriptionInformationDict(self, title):
+    """
+    Return a dict with informations about Hosting subscription
+    """
+    related_instance_link_list = self._getRelatedInstanceLink(title)
+    related_instance_link = None
+    # Get root instance
+    for link in related_instance_link_list:
+      instance = self._curl(link)
+      if title == instance['title']:
+        related_instance_link = link
+        break
+    # Return information dict
+    if related_instance_link:
+      related_instance = self._curl(related_instance_link)
+      return {
+        'title': related_instance['title'],
+        'status': related_instance['status'],
+        'software_url': related_instance['_links']['http://slapos.org/reg/release'],
+        'software_type': related_instance['software_type'],
+        'computer_guid': related_instance['sla']['computer_guid']
+      }
+    else:
+      return None
+  
\ No newline at end of file

erp5/util/testnode/UnitTestRunner.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from datetime import datetime,timedelta
+import os
+import subprocess
+import sys
+import time
+import glob
+import SlapOSControler
+import json
+import time
+import shutil
+import logging
+import string
+import random
+from ProcessManager import SubprocessError, ProcessManager, CancellationError
+from subprocess import CalledProcessError
+from NodeTestSuite import SlapOSInstance
+from Updater import Updater
+from erp5.util import taskdistribution
+
+class UnitTestRunner():
+  def __init__(self, testnode):
+    self.testnode = testnode
+    self.slapos_controler = SlapOSControler.SlapOSControler(
+                                  self.testnode.working_directory,
+                                  self.testnode.config,
+                                  self.testnode.log)
+
+  def _prepareSlapOS(self, working_directory, slapos_instance, log,
+          create_partition=1, software_path_list=None, **kw):
+    """
+    Launch slapos to build software and partitions
+    """
+    slapproxy_log = os.path.join(self.testnode.config['log_directory'],
+                                  'slapproxy.log')
+    log('Configured slapproxy log to %r' % slapproxy_log)
+    reset_software = slapos_instance.retry_software_count > 10
+    if reset_software:
+      slapos_instance.retry_software_count = 0
+    log('testnode, retry_software_count : %r' % \
+             slapos_instance.retry_software_count)
+    self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
+       process_manager=self.testnode.process_manager, reset_software=reset_software,
+       software_path_list=software_path_list)
+    self.testnode.process_manager.supervisord_pid_file = os.path.join(\
+         self.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+    method_list= ["runSoftwareRelease"]
+    if create_partition:
+      method_list.append("runComputerPartition")
+    for method_name in method_list:
+      slapos_method = getattr(self.slapos_controler, method_name)
+      log("Before status_dict = slapos_method(...)")
+      status_dict = slapos_method(self.testnode.config,
+                                  environment=self.testnode.config['environment'],
+                                 )
+      log(status_dict)
+      log("After status_dict = slapos_method(...)")
+      if status_dict['status_code'] != 0:
+         slapos_instance.retry = True
+         slapos_instance.retry_software_count += 1
+         raise SubprocessError(status_dict)
+      else:
+         slapos_instance.retry_software_count = 0
+    return status_dict
+
+  def prepareSlapOSForTestNode(self, test_node_slapos):
+    """
+    We will build slapos software needed by the testnode itself,
+    like the building of selenium-runner by default
+    """
+    return self._prepareSlapOS(self.testnode.config['slapos_directory'],
+              test_node_slapos, self.testnode.log, create_partition=0,
+              software_path_list=self.testnode.config.get("software_list"))
+
+  def prepareSlapOSForTestSuite(self, node_test_suite):
+    """
+    Build softwares needed by testsuites
+    """
+    log = self.testnode.log
+    if log is None:
+      log = self.testnode.log
+    return self._prepareSlapOS(node_test_suite.working_directory,
+              node_test_suite, log,
+              software_path_list=[node_test_suite.custom_profile_path])
+
+  def runTestSuite(self, node_test_suite, portal_url, log=None):
+    config = self.testnode.config
+    parameter_list = []
+    run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" % \
+        self.slapos_controler.instance_root)
+    if not len(run_test_suite_path_list):
+      raise ValueError('No runTestSuite provided in installed partitions.')
+    run_test_suite_path = run_test_suite_path_list[0]
+    run_test_suite_revision = node_test_suite.revision
+    # Deal with Shebang size limitation
+    invocation_list = self.testnode._dealShebang(run_test_suite_path)
+    invocation_list.extend([run_test_suite_path,
+                           '--test_suite', node_test_suite.test_suite,
+                           '--revision', node_test_suite.revision,
+                           '--test_suite_title', node_test_suite.test_suite_title,
+                           '--node_quantity', config['node_quantity'],
+                           '--master_url', portal_url])
+    firefox_bin_list = glob.glob("%s/soft/*/parts/firefox/firefox-slapos" % \
+        config["slapos_directory"])
+    if len(firefox_bin_list):
+      parameter_list.append('--firefox_bin')
+    xvfb_bin_list = glob.glob("%s/soft/*/parts/xserver/bin/Xvfb" % \
+        config["slapos_directory"])
+    if len(xvfb_bin_list):
+      parameter_list.append('--xvfb_bin')
+    supported_paramater_set = self.testnode.process_manager.getSupportedParameterSet(
+                           run_test_suite_path, parameter_list)
+    if '--firefox_bin' in supported_paramater_set:
+      invocation_list.extend(["--firefox_bin", firefox_bin_list[0]])
+    if '--xvfb_bin' in supported_paramater_set:
+      invocation_list.extend(["--xvfb_bin", xvfb_bin_list[0]])
+    # TODO : include testnode correction ( b111682f14890bf )
+    if hasattr(node_test_suite,'additional_bt5_repository_id'):
+      additional_bt5_path = os.path.join(
+              node_test_suite.working_directory,
+              node_test_suite.additional_bt5_repository_id)
+      invocation_list.extend(["--bt5_path", additional_bt5_path])
+    # From this point, test runner becomes responsible for updating test
+    # result. We only do cleanup if the test runner itself is not able
+    # to run.
+    SlapOSControler.createFolder(node_test_suite.test_suite_directory,
+                                 clean=True)
+    self.testnode.process_manager.spawn(*invocation_list,
+                          cwd=node_test_suite.test_suite_directory,
+                          log_prefix='runTestSuite', get_output=False)
+
+  def getRelativePathUsage(self):
+    """
+    Used by the method testnode.constructProfile() to know
+    if the software.cfg have to use relative path or not.
+    """
+    return False

erp5/util/testnode/Updater.py

@@ -32,7 +32,7 @@ import subprocess
 import sys
 import threading
 
-from testnode import SubprocessError
+from ProcessManager import SubprocessError
 
 SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
 SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
@@ -103,6 +103,30 @@ class Updater(object):
   def _git(self, *args, **kw):
     return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
 
+  def git_update_server_info(self):
+    return self._git('update-server-info', '-f')
+
+  def git_create_repository_link(self):
+    """ Create a link in depository to the ".git" directory.
+        ex:
+        for "../erp5/.git"
+        "../erp5/erp5.git"->"../erp5/.git" will be created.
+    """
+    git_repository_path = os.path.join(self.getRepositoryPath(), '.git')
+    name = os.path.basename(os.path.normpath(self.getRepositoryPath()))
+    git_repository_link_path = os.path.join(self.getRepositoryPath(), '%s.git' %name)
+    self.log("checking link %s -> %s.."
+                %(git_repository_link_path,git_repository_path))
+    if ( not os.path.lexists(git_repository_link_path) and \
+         not os.path.exists(git_repository_link_path) ):
+      try:
+        os.symlink(git_repository_path, git_repository_link_path)
+        self.log("link: %s -> %s created"
+                %(git_repository_link_path,git_repository_path))
+      except:
+        self.log("Cannot create link from %s -> %s"
+                %(git_repository_link_path,git_repository_path))
+  
   def _git_find_rev(self, ref):
     try:
       return self._git_cache[ref]
@@ -213,3 +237,4 @@ class Updater(object):
     else:
       raise NotImplementedError
     self._path_list += path_list
+    self.git_update_server_info()

erp5/util/testnode/Utils.py

+import sys
+import json
+import shutil
+import string
+from random import choice
+
+def deunicodeData(data):
+  if isinstance(data, list):
+    new_data = []
+    for sub_data in data:
+      new_data.append(deunicodeData(sub_data))
+  elif isinstance(data, unicode):
+    new_data = data.encode('utf8')
+  elif isinstance(data, dict):
+    new_data = {}
+    for key, value in data.iteritems():
+      key = deunicodeData(key)
+      value = deunicodeData(value)
+      new_data[key] = value
+  else:
+    new_data = data
+  return new_data
\ No newline at end of file

erp5/util/testnode/__init__.py

@@ -74,15 +74,17 @@ def main(*args):
   config.optionxform = str
   config.readfp(parsed_argument.configuration_file[0])
   for key in ('slapos_directory','working_directory','test_suite_directory',
-              'log_directory','run_directory','proxy_host','proxy_port',
-              'git_binary','zip_binary','node_quantity','test_node_title',
-              'ipv4_address','ipv6_address','test_suite_master_url',
+              'log_directory','run_directory', 'srv_directory', 'proxy_host',
+              'software_directory',
+              'proxy_port', 'git_binary','zip_binary','node_quantity',
+              'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
               'slapgrid_partition_binary','slapgrid_software_binary',
-              'slapproxy_binary', 'httpd_ip', 'httpd_port'):
+              'slapproxy_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
+              'computer_id', 'server_url'):
     CONFIG[key] = config.get('testnode',key)
 
   for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
-      'log_directory', 'run_directory'):
+      'log_directory', 'run_directory', 'srv_directory', 'software_directory'):
     d = CONFIG[key]
     if not os.path.isdir(d):
       raise ValueError('Directory %r does not exists.' % d)
@@ -107,6 +109,6 @@ def main(*args):
   if 'software_list' in config.sections():
     CONFIG['software_list'] = filter(None,
         config.get("software_list", "path_list").split(","))
-
+  
   testnode = TestNode(logger.info, CONFIG)
   testnode.run()