Commit d346640b authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼 Committed by Lu Xu

erp5.util: add a new test type to do real requests to the slapos master

This was previously known as "SlapOS Agent".
parent bf89c93a
......@@ -489,6 +489,12 @@ class TaskDistributor(RPCRetry):
"""
return self._retryRPC('getSlaposHateoasUrl')
def getServiceList(self, test_suite_title):
"""
Returns the list of services needed to run the test
"""
return self._retryRPC('getServiceList', (test_suite_title,))
def createTestResult(self, revision, test_name_list, node_title,
allow_restart=False, test_title=None, project_title=None):
"""
......
This diff is collapsed.
......@@ -386,10 +386,11 @@ Require valid-user
self.error_message = test_configuration['error_message']
self.randomized_path = test_configuration['randomized_path']
if not self.launchable:
logger.info("Test suite %s is not actually launchable"
" with the current cluster configuration.", node_test_suite.test_suite_title)
logger.info("ERP5 Master indicates : %s", self.error_message)
return {'status_code' : 1}
error_message = ("Test suite %s is not actually launchable"
" with the current cluster configuration.\n"
"ERP5 Master indicates : %s" % (node_test_suite.test_suite_title, self.error_message))
logger.info(error_message)
return {'status_code' : 1, 'error_message' : error_message}
configuration_list = test_configuration['configuration_list']
configuration = configuration_list[0]
......@@ -451,8 +452,9 @@ Require valid-user
self._comeBackFromDummySlapOS()
if self.remainSoftwareToInstall() :
# All softwares are not installed, however maxtime is elapsed, that's a failure.
logger.error("All softwares are not installed.")
return {'status_code' : 1}
error_message = "All softwares are not installed."
logger.error(error_message)
return {'status_code' : 1, 'error_message' : error_message}
logger.debug("All software installed.")
# even if we re-use existing setup we need proper configuration applied
......@@ -469,11 +471,12 @@ Require valid-user
purge_previous_instance = not self.use_existing_setup)
logger.debug("Scalability instance requested.")
except Exception as e:
logger.error("Error creating instance: " + str(e))
return {'status_code' : 1}
error_message = "Error creating instance: " + str(e)
logger.error(error_message)
return {'status_code' : 1, 'error_message' : error_message}
return {'status_code' : 0}
return {'status_code' : 1}
return {'status_code' : 1, 'error_message' : "Software installation too long or error(s) are present during SR install."}
def makeSuite(self, test_suite, location_list, **kwargs):
import imp
......
from __future__ import print_function
import datetime
import feedparser
import json
import traceback
import time
......@@ -71,6 +72,8 @@ class SlapOSMasterCommunicator(object):
self.slap_order = slap_order
self.slap_supply = slap_supply
self.hateoas_navigator = self.slap._hateoas_navigator
self.message_history = []
self.computer_guid = ""
if url is not None and \
url.startswith(SOFTWARE_PRODUCT_NAMESPACE):
......@@ -79,7 +82,7 @@ class SlapOSMasterCommunicator(object):
try:
url = product.__getattr__(url[len(SOFTWARE_PRODUCT_NAMESPACE):])
except AttributeError as e:
logger.warning('Error on get software release: %s ', e.message)
logger.warning("Error on get software release: {}".format(e))
self.url = url
......@@ -115,6 +118,7 @@ class SlapOSMasterCommunicator(object):
partition_reference=self.name,
shared=shared,
state=state,
software_type=software_type,
**self.request_kw)
@retryOnNetworkFailure
......@@ -183,16 +187,18 @@ class SlapOSMasterCommunicator(object):
message_list = []
try:
for instance in self.getInstanceUrlList():
# we need to explicitly encode as utf-8 the unicode string we get
instance["text_content"] = instance["text_content"].encode('utf8')
logger.info('in _getInstanceState, viewing instance')
logger.info(instance)
news = instance['SoftwareInstance_getNewsDict']
state = INSTANCE_STATE_UNKNOWN
monitor_information_dict = {}
is_slave = instance['portal_type'] == "Slave Instance"
if is_slave:
if len(instance['getConnectionXmlAsDict']) > 0:
state = INSTANCE_STATE_STARTED
# XXX for now consider a slave as always ready because in ORS software
# there is no information published in the slave
#if len(instance['getConnectionXmlAsDict']) > 0:
state = INSTANCE_STATE_STARTED
else:
# not slave
instance_state = news
......@@ -300,7 +306,6 @@ class SlapOSTester(SlapOSMasterCommunicator):
self.request_kw = json.loads(request_kw)
else:
self.request_kw = request_kw
self.message_history = []
def getInfo(self):
info = ""
......@@ -323,12 +328,13 @@ class SlapOSTester(SlapOSMasterCommunicator):
def requestInstanceStop(self, instance_title=None, request_kw=None, shared=False, software_type="RootSoftwareInstance"):
self.instance = self._request(INSTANCE_STATE_STOPPED, instance_title, request_kw, shared, software_type)
def requestInstanceDestroy(self, instance_title=None, request_kw=None, shared=False):
def requestInstanceDestroy(self):
# TODO remove this function
self.destroyInstance()
def waitInstanceStarted(self, instance_title=None):
if not instance_title:
instance_title = self.name
self.destroyInstance(instance_title)
def waitInstanceStarted(self, instance_title):
error_message = self._waitInstance(instance_title, INSTANCE_STATE_STARTED)["error_message"]
if error_message is not None:
logger.error(error_message)
......@@ -349,6 +355,13 @@ class SlapOSTester(SlapOSMasterCommunicator):
logger.error(error_message)
raise ValueError(error_message)
def getInstanceParameterDict(self):
for instance in self.getInstanceUrlList():
if instance["title"] == self.name:
return instance["getConnectionXmlAsDict"]
return {}
def getMasterFrontendDict(self):
def getInstanceGuid():
try:
......@@ -365,7 +378,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
pass
start_time = time.time()
while not getInstanceGuid() and time.time()-start_time < 60*5:
sleep(60)
time.sleep(60)
return {'instance_guid' : getInstanceGuid(), 'frontend_master_ipv6' : frontend_master_ipv6}
# XXX TODO
......@@ -406,16 +419,15 @@ class SlapOSTester(SlapOSMasterCommunicator):
'frontend-url-list' : frontend_url_list, \
'balancer-user-v6': balancer_user_v6 }
def destroyInstance(self, instance_title):
self.name = instance_title
def destroyInstance(self):
instance_url_list = self.getInstanceUrlList()
if instance_url_list:
for instance in instance_url_list:
if instance["title"] != instance_title:
if instance["title"] != self.name:
self._request(INSTANCE_STATE_DESTROYED, instance["title"])
else:
root_instance = instance
logger.info("Going to destroy root partition: " + str(instance_title))
logger.info("Going to destroy root partition: " + str(self.name))
self._request(INSTANCE_STATE_DESTROYED, root_instance["title"])
else:
logger.info("Instance not found")
......@@ -537,7 +549,7 @@ class SoftwareReleaseTester(SlapOSTester):
deadline = self.deadline
if deadline < now and deadline is not None:
raise TestTimeout(self.state)
raise Exception("Test timeout (current state is {}).".format(self.state))
_, _, next_state, software_state, instance_state = self.transition_dict[
self.state]
......
......@@ -32,14 +32,10 @@ import logging
from . import logger
from .ProcessManager import SubprocessError, format_command
from .SlapOSControler import SlapOSControler
from .Utils import createFolder
from .Utils import createFolder, dealShebang
from slapos.grid.utils import md5digest
def dealShebang(run_test_suite_path):
with open(run_test_suite_path) as f:
if f.read(2) == '#!':
return f.readline().split(None, 1)
return []
class UnitTestRunner(object):
......@@ -203,6 +199,7 @@ class UnitTestRunner(object):
log_prefix='runTestSuite',
output_replacers=(hide_distributor_url,),
get_output=False)
return {'status_code' : 0}
def getRelativePathUsage(self):
"""
......
......@@ -10,6 +10,12 @@ def createFolder(folder, clean=False):
rmtree(folder)
os.mkdir(folder)
def dealShebang(run_test_suite_path):
with open(run_test_suite_path) as f:
if f.read(2) == '#!':
return f.readline().split(None, 1)
return []
if six.PY3:
def deunicodeData(data):
return data
......
......@@ -38,6 +38,7 @@ from subprocess import CalledProcessError
from .Updater import Updater
from .NodeTestSuite import NodeTestSuite, SlapOSInstance
from .ScalabilityTestRunner import ScalabilityTestRunner
from .RealRequestRunner import RealRequestRunner
from .UnitTestRunner import UnitTestRunner
from .Utils import deunicodeData
from .. import taskdistribution
......@@ -49,6 +50,7 @@ PROFILE_PATH_KEY = 'profile_path'
test_type_registry = {
'UnitTest': UnitTestRunner,
'ScalabilityTest': ScalabilityTestRunner,
'SlapOSAgentTest': RealRequestRunner
}
class TestNode(object):
......@@ -106,13 +108,13 @@ class TestNode(object):
# Absolute path to relative path
software_config_path = os.path.join(repository_path, profile_path)
if use_relative_path :
if use_relative_path:
from_path = os.path.join(self.working_directory,
node_test_suite.reference)
software_config_path = os.path.relpath(software_config_path, from_path)
# Construct sections
if not(buildout_section_id is None):
if buildout_section_id is not None:
# Absolute path to relative
if use_relative_path:
from_path = os.path.join(self.working_directory,
......@@ -320,7 +322,7 @@ shared = true
testnode_software_successfully_built = True
logger.info("Will now skip build of testnode software")
# Clean-up test suites
self.purgeOldTestSuite(test_suite_data)
#self.purgeOldTestSuite(test_suite_data)
for test_suite in test_suite_data:
node_test_suite = self.getNodeTestSuite(
test_suite.pop("test_suite_reference"))
......@@ -354,6 +356,8 @@ shared = true
generated_config = taskdistributor.generateConfiguration(
node_test_suite.test_suite_title)
json_data = json.loads(generated_config)
logger.info("DEBUG JSON")
logger.info(json_data)
cluster_configuration = deunicodeData(json_data['configuration_list'][0])
node_test_suite.edit(cluster_configuration=cluster_configuration)
# Now prepare the installation of SlapOS and create instance
......@@ -364,35 +368,26 @@ shared = true
# should be at the same revision, so it is safe to prune orphan
# objects now.
git_gc_auto()
def report_error(error_message):
test_result.reportFailure(
stdout=error_message
)
logger.error(error_message)
raise ValueError(error_message)
if status_dict['status_code'] == 1:
report_error(status_dict.get('error_message') or "Error during prepareSlapOSForTestSuite")
# Give some time so computer partitions may start
# as partitions can be of any kind we have and likely will never have
# a reliable way to check if they are up or not ...
time.sleep(20)
# XXX: Do not switch according to the test type. IOW, the
# following code must be moved to the test type class.
if my_test_type == 'UnitTest':
runner.runTestSuite(node_test_suite, portal_url)
elif my_test_type == 'ScalabilityTest':
error_message = None
# A problem is appeared during runTestSuite
if status_dict['status_code'] == 1:
error_message = "Software installation too long or error(s) are present during SR install."
else:
status_dict = runner.runTestSuite(node_test_suite, portal_url)
# A problem is appeared during runTestSuite
if status_dict['status_code'] == 1:
error_message = status_dict['error_message']
# If an error is appeared
if error_message:
test_result.reportFailure(
stdout=error_message
)
logger.error(error_message)
raise ValueError(error_message)
else:
raise NotImplementedError
# break the loop to get latest priorities from master
runner.runTestSuite(node_test_suite, portal_url)
status_dict = runner.runTestSuite(node_test_suite, portal_url)
# A problem is appeared during runTestSuite
if status_dict['status_code'] == 1:
report_error(status_dict.get('error_message') or 'Error during runTestSuite')
break
except (SubprocessError, CalledProcessError, ConnectionError) as e:
logger.exception("")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment