NEO: new stress SR

software/neoppod/stress-testing/runTestSuite.in

+#!{{parameter_dict['runTestSuite_py']}}
+from __future__ import print_function
+import argparse, curses, json, os, subprocess, sys, traceback
+from time import gmtime, strftime, time
+# These are the 2 modules to reuse when using ERP5 for managing test bots.
+# What we do here is currently too new to reuse more from testsuite.
+from erp5.util import taskdistribution
+from erp5.util.testsuite import format_command
+from zc.buildout.buildout import Buildout
+
+{% set vm = parameter_dict['vm'] -%}
+
+dist_list = {{vm['dists'].split()}}
+publish = {{slapparameter_dict.get('publish')}}
+
+
+class DummyTestResult:
+
+  class DummyTestResultLine:
+
+    def stop(self, **kw):
+      with open(self.name + '.json', 'w') as f:
+        json.dump(kw, f)
+
+  done = 0
+
+  def __init__(self, test_name_list):
+    self.test_name_list = test_name_list
+
+  def start(self):
+    test_result_line = self.DummyTestResultLine()
+    try:
+      test_result_line.name = self.test_name_list[self.done]
+    except IndexError:
+      return
+    self.done += 1
+    return test_result_line
+
+
+def main():
+  os.environ.update({k: v.strip() % os.environ
+    for k, v in {{parameter_dict['environment'].items()}}})
+
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', type=int,
+                      help='Number of CPUs to use for the VM')
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+
+  args = parser.parse_args()
+
+  test_title = args.test_suite_title or args.test_suite
+  if args.master_url:
+    tool = taskdistribution.TaskDistributor(args.master_url)
+    test_result = tool.createTestResult(args.revision,
+                                        dist_list,
+                                        args.test_node_title,
+                                        test_title=test_title,
+                                        project_title=args.project_title)
+    if test_result is None:
+      return
+  else:
+    test_result = DummyTestResult(dist_list)
+
+  Buildout._setup_logging = lambda self: None
+
+  # XXX: fix install of slapos.libnetworkcache in offline mode
+  import slapos.libnetworkcache
+  from zc.buildout.easy_install import buildout_and_setuptools_path
+  path, = slapos.__path__
+  buildout_and_setuptools_path.append(os.path.dirname(path))
+  ###
+
+  fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666)
+  try:
+    os.write(fd, """\
+[buildout]
+offline = true
+relative-paths = true
+""")
+    Buildout('buildout.cfg', ()).bootstrap(None)
+    os.write(fd, """\
+extends = {{parameter_dict['profile_base_location']}}/stress.cfg
+develop-eggs-directory = {{buildout['develop-eggs-directory']}}
+eggs-directory = {{buildout['eggs-directory']}}
+
+[stress]
+environment =
+vm = {{vm['location']}}
+mount.neoppod = {{parameter_dict['neoppod']}}
+""")
+  finally:
+    os.close(fd)
+  partdir = os.path.join('parts', 'stress')
+  stderr_write = sys.stderr.write
+
+  while 1:
+    test_result_line = test_result.start()
+    if not test_result_line:
+      break
+    dist = test_result_line.name
+
+    cmd = [os.path.join('bin', 'buildout'),
+      'stress:dist=' + dist,
+    ]
+    if args.node_quantity:
+      cmd.append('stress:smp=%s' % args.node_quantity)
+    status_dict = {'command': format_command(*cmd)}
+    print('$', status_dict['command'])
+
+    # Wanted on test result lines:
+    #  status: UNKNOWN in case of buildout failure
+    #  output: stress check if any
+    #  error: buildout traceback or VM's disk usage
+
+    start = time()
+    try:
+      try:
+        p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+        stderr = []
+        while 1:
+          line = p.stderr.readline()
+          if not line:
+            break
+          stderr_write(line)
+          stderr.append(line)
+        returncode = p.wait()
+      finally:
+        end = time()
+      del p
+      status_dict['stderr'] = ''.join(stderr)
+
+      if not returncode:
+        try:
+          with open(os.path.join(partdir, 'status')) as f:
+            status = int(f.read())
+        except Exception:
+          pass
+        else:
+          status_dict['test_count'] = 1
+          try:
+            curses.setupterm('screen')
+            rmkx = curses.tigetstr('rmkx')
+            with open(os.path.join(partdir, 'stdout')) as f:
+              stdout = f.read()
+            status_dict['stdout'] = stdout[stdout.rindex(rmkx)+len(rmkx):]
+          except Exception:
+            status = -1
+          if status:
+            status_dict['error_count'] = 1
+
+    except Exception:
+      status_dict.setdefault('stderr', traceback.format_exc())
+
+    test_result_line.stop(
+      date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      duration = end - start,
+      **status_dict)
+
+
+if __name__ == "__main__":
+    main()

software/neoppod/stress-testing/software.cfg

+[buildout]
+extends =
+  ../software.cfg
+  ../../../component/vm-img/debian.cfg
+parts =
+  slapos-cookbook
+  template
+  neoppod-develop
+  neolog
+
+[template]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template =
+  inline:[buildout]
+  eggs-directory = ${buildout:eggs-directory}
+  develop-eggs-directory = ${buildout:develop-eggs-directory}
+  parts = runTestSuite
+  [slap-configuration]
+  recipe = slapos.cookbook:slapconfiguration.serialised
+  computer = $${slap-connection:computer-id}
+  partition = $${slap-connection:partition-id}
+  url = $${slap-connection:server-url}
+  key = $${slap-connection:key-file}
+  cert = $${slap-connection:cert-file}
+  [directory]
+  recipe = slapos.cookbook:mkdirectory
+  bin = $${buildout:directory}/bin
+  [runTestSuite]
+  recipe = slapos.recipe.template:jinja2
+  rendered = $${directory:bin}/$${:_buildout_section_name_}
+  template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
+  mode = 0755
+  context =
+    section buildout buildout
+    section parameter_dict runTestSuite-parameters
+    key slapparameter_dict slap-configuration:configuration
+  [runTestSuite-parameters]
+  environment = {{dumps(environment)}}
+  vm = {{dumps(vm)}}
+  runTestSuite_py = ${buildout:bin-directory}/${runTestSuite_py:interpreter}
+  profile_base_location = ${:_profile_base_location_}
+  neoppod = ${neoppod-repository:location}
+context =
+  section environment vm-run-environment
+  section vm vm-debian
+
+[runTestSuite_py]
+eggs +=
+  zc.buildout
+  slapos.libnetworkcache
+
+# Just in case one wants to inspect logs on the testnode itself.
+[neolog]
+recipe = zc.recipe.egg
+eggs =
+  ${msgpack-python:egg}
+  neoppod
+scripts = neolog
+
+[neoppod-repository]
+# When run from erp5.util:testnode (and this always the case), shared=true is
+# added to this section and the path in .git/objects/info/alternates is outside
+# the 'neoppod' mount point of the VM. Let's mount the testnode working copy
+# directly. Comment the following 2 lines if 'repository' is a URL.
+recipe =
+location = ${:repository}
+
+[vm-debian]
+# Mainly for logs, otherwise 2G would enough (<100M for the SR).
+size = 8Gi
+late-command +=
+  echo tmpfs /tmp tmpfs mode=1777,size=90% 0 0 >>/etc/fstab
+packages +=
+# dependencies to build a SlapOS SR without slapos-node
+  ca-certificates file g++ libc6-dev make patch python
+# speed up build by using the following components from the OS
+  git liblzma-dev libssl-dev pkg-config python-dev
+# extra requirements for NEO
+  libnetfilter-queue-dev nftables
+# extra requirements for this SR
+  screen xz-utils
+
+[versions]
+slapos.recipe.template = 4.3

software/neoppod/stress-testing/stress

+#!/usr/bin/python
+import curses, os, subprocess, sys, tempfile, threading, time
+from functools import partial
+curses.setupterm()
+
+result = partial(os.path.join, sys.argv[1])
+duration = int(sys.argv[2])
+
+cmd = sys.argv[3:]
+cmd[cmd.index('--')] = 'run'
+cmd[:0] = 'bin/py', 'parts/neoppod-repository/tools/stress', '-l', '../log'
+
+p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+time.sleep(10)
+p.stdin.write(curses.tigetstr("kf1"))
+t = threading.Timer(duration, p.stdin.write, ('q',))
+t.daemon = True
+try:
+    t.start()
+    returncode = p.wait()
+finally:
+    t.cancel()
+
+with open(result('status'), 'wb') as f:
+    f.write(str(returncode))
+
+if returncode:
+    subprocess.call(
+        ('tar', '-caf', result('log.tar.xz'), 'log'),
+        cwd=os.pardir)
+    subprocess.call(
+        ('tar', '-caf', result('db.tar.xz'), 'neo_tests'),
+        cwd=tempfile.gettempdir())

software/neoppod/stress-testing/stress.cfg

+[buildout]
+extends =
+  ../../../component/vm-img/debian.cfg
+parts =
+  stress
+
+[stress]
+<= vm-run-base
+smp = 8
+# NEO processes take ~2G with default settings,
+# the rest is mainly for the DB (which is in tmpfs)
+mem = 4096
+duration = 3600
+stress-args =
+run-args = -L .24 -r 0
+mount.slapos = ${:_profile_base_location_}/../../..
+command =
+  mkdir log software
+  cd software
+  cat <<EOF >buildout.cfg
+  [buildout]
+  extends =
+    /mnt/slapos/software/neoppod/software.cfg
+    /mnt/slapos/software/neoppod/sqlite-mixin.cfg
+  parts =
+    neoppod-develop
+    neoppod
+  develop = /mnt/slapos
+  extensions -= slapos.rebootstrap
+  [slapos-cookbook-develop]
+  recipe =
+  setup =
+  [neoppod-repository]
+  repository = /mnt/neoppod
+  shared = true
+  [neoppod]
+  eggs +=
+    NetfilterQueue
+    gevent
+  interpreter = py
+  [versions]
+  slapos.cookbook =
+  NetfilterQueue = 0.8.1
+  greenlet = 0.4.15
+  gevent = 1.3.7
+  cffi = 1.11.5
+  # use the following components from the OS
+  # and don't build dependencies for nothing
+  [git]
+  recipe =
+  location = /usr
+  environment =
+  configure-options =
+  [openssl]
+  recipe =
+  location = /usr
+  configure-options =
+  make-options =
+  make-targets =
+  environment =
+  [perl]
+  recipe =
+  location = /usr
+  configure-command =
+  environment =
+  [pkgconfig]
+  recipe =
+  location = /usr
+  environment =
+  [python2.7]
+  recipe =
+  location = /usr
+  environment =
+  [xz-utils]
+  recipe =
+  location = /usr
+  EOF
+  python -S /mnt/buildout/bin/buildout bootstrap
+  MAKEFLAGS=-j${:smp} bin/buildout
+  df ~ /tmp >&2
+  screen -L -Logfile $PARTDIR/stdout -D -m sudo /mnt/slapos/software/neoppod/stress-testing/stress $PARTDIR ${:duration} ${:stress-args} -- ${:run-args}
+  df ~ /tmp >&2

stack/slapos.cfg

@@ -142,7 +142,7 @@ slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.19
 slapos.rebootstrap = 4.2
-slapos.recipe.build = 0.41
+slapos.recipe.build = 0.42
 slapos.recipe.cmmi = 0.11
 slapos.toolbox = 0.95
 stevedore = 1.21.0