Add support for software instanciation.

Reuse erp5.util.taskdistribution for better TaskDistributionTool
integration (adds a dependency, removes 2 entry points): it is possible to
create several test agents with same configuration, each testing a subset
of the entire test suite. Especially useful if/when test agents gets the
extra task of monitoring commits on software releases, and they locally
build & instanciate the SR they test.
Use a medium-lived process rather than a short-lived one, to drop state
loading & saving complexity.
Allow a single test agent to run several concurrent tests.
Cleanup pidfile, even when exceptions occur.
Factorise code used to test each step.
Rework configuration file layout, to better use magic "[DEFAULT]" section.
Take keys & certs from configuration file, allowing for per-test certs.
Remove the need for storing using login & password by reusing x509 auth.

setup.py

@@ -45,14 +45,13 @@ setup(name=name,
       extras_require = {
         'lampconfigure':  ["mysql-python"], #needed for MySQL Database access
         'zodbpack': ['ZODB3'], # needed to play with ZODB
+        'agent': ['erp5.util'],
       },
       zip_safe=False, # proxy depends on Flask, which has issues with
                       # accessing templates
       entry_points={
         'console_scripts': [
-          'agent = slapos.agent.agent:main',
-          'report_start = slapos.agent.report_start:main',
-          'report_stop = slapos.agent.report_stop:main',
+          'agent = slapos.agent.agent:main [agent]',
           'clouddestroy = slapos.cloudmgr.destroy:main',
           'cloudgetprivatekey = slapos.cloudmgr.getprivatekey:main',
           'cloudgetpubliciplist = slapos.cloudmgr.getpubliciplist:main',

slapos/agent/report_start.py

-import ConfigParser, argparse, pprint, socket, sys, time, xmlrpclib, json
-
-def safeRpcCall(function, *args):
-  retry = 64
-  xmlrpc_arg_list = []
-  for argument in args:
-    if isinstance(argument, dict):
-      argument = dict([(x, isinstance(y,str) and xmlrpclib.Binary(y) or y) \
-           for (x,y) in argument.iteritems()])
-    xmlrpc_arg_list.append(argument)
-  while True:
-    try:
-      return function(*xmlrpc_arg_list)
-    except (socket.error, xmlrpclib.ProtocolError), e:
-      print >>sys.stderr, e
-      pprint.pprint(args, file(function._Method__name, 'w'))
-      time.sleep(retry)
-      retry += retry >> 1
-
-def main(*args):
-  parser = argparse.ArgumentParser()
-  parser.add_argument("configuration_file", nargs=1, type=argparse.FileType(),
-      help="Slap Test Agent configuration file.")
-  if args == ():
-    argument_option_instance = parser.parse_args()
-  else:
-    argument_option_instance = \
-      parser.parse_args(list(args))
-  configuration_file = argument_option_instance.configuration_file[0]
-  configuration = ConfigParser.SafeConfigParser()
-  configuration.readfp(configuration_file)
-  master_url = configuration.get("agent", "report_url")
-  if master_url[-1] != '/':
-    master_url += '/'
-  master = xmlrpclib.ServerProxy("%s%s" %
-            (master_url, 'portal_task_distribution'),
-            allow_none=1)
-  assert master.getProtocolRevision() == 1
-  software_list = json.loads(configuration.get("agent", "software_list"))
-  test_result = safeRpcCall(master.createTestResult,
-    "SlapOS Test", "", software_list,
-    True, "SlapOS Test", "SlapOS Test Agent", "ViFiB Project")
-  test_result_path, revision = test_result
-  state = ConfigParser.SafeConfigParser()
-  state.add_section("path")
-  exclude_list = software_list[:]
-  for software in software_list:
-    exclude_list.remove(software)
-    test_path, test_name = safeRpcCall(master.startUnitTest,
-      test_result_path, exclude_list)
-    state.set("path", test_name, test_path)
-    exclude_list.append(software)
-  path_file = configuration.get("agent", "path_file")
-  state.write(open(path_file, "w"))
-
-if __name__ == "__main__":
-  main()

slapos/agent/report_stop.py

-import ConfigParser, argparse, pprint, socket, sys, time, xmlrpclib, json, os
-from datetime import datetime
-
-def safeRpcCall(function, *args):
-  retry = 64
-  xmlrpc_arg_list = []
-  for argument in args:
-    if isinstance(argument, dict):
-      argument = dict([(x, isinstance(y,str) and xmlrpclib.Binary(y) or y) \
-           for (x,y) in argument.iteritems()])
-    xmlrpc_arg_list.append(argument)
-  while True:
-    try:
-      return function(*xmlrpc_arg_list)
-    except (socket.error, xmlrpclib.ProtocolError), e:
-      print >>sys.stderr, e
-      pprint.pprint(args, file(function._Method__name, 'w'))
-      time.sleep(retry)
-      retry += retry >> 1
-
-def main(*args):
-  parser = argparse.ArgumentParser()
-  parser.add_argument("configuration_file", nargs=1, type=argparse.FileType(),
-      help="Slap Test Agent configuration file.")
-  if args == ():
-    argument_option_instance = parser.parse_args()
-  else:
-    argument_option_instance = \
-      parser.parse_args(list(args))
-  configuration_file = argument_option_instance.configuration_file[0]
-  configuration = ConfigParser.SafeConfigParser()
-  configuration.readfp(configuration_file)
-  master_url = configuration.get("agent", "report_url")
-  software_list = json.loads(configuration.get("agent", "software_list"))
-  if master_url[-1] != '/':
-    master_url += '/'
-  master = xmlrpclib.ServerProxy("%s%s" %
-            (master_url, 'portal_task_distribution'),
-            allow_none=1)
-  assert master.getProtocolRevision() == 1
-
-  log_directory = configuration.get("agent", "log_directory")
-  logfile_path = os.path.join(log_directory, "agent-%s.log" % datetime.strftime(datetime.now(), "%Y%m%d"))
-  logfile = open(logfile_path, 'r')
-  logline_list = logfile.readlines()
-
-  path_file = configuration.get("agent", "path_file")
-  state = ConfigParser.SafeConfigParser()
-  state.readfp(open(path_file))
-  for software in software_list:
-    test_path = state.get("path", software)
-    success, failure = 0, 0
-    log = ''
-    for logline in logline_list:
-      success_pattern = "Successfully installed %s" % software
-      failure_pattern = "Failed to install %s" % software
-      if success_pattern in logline:
-        success = success + 1
-        log = log + logline
-      if failure_pattern in logline:
-        failure = failure + 1
-        log = log + logline
-    safeRpcCall(master.stopUnitTest, test_path,
-      {
-        "status_code": 0,
-        "stderr": log,
-        "duration": 0,
-        "test_count": success + failure,
-        "failure_count": failure
-      })
-
-if __name__ == "__main__":
-  main()