software/cythonplus-dev: Add software release for cython+ development

component/cythonplus/buildout.cfg

+[buildout]
+extends =
+  ../defaults.cfg
+  ../git/buildout.cfg
+  ../numpy/buildout.cfg
+
+parts = cythonplus_env.sh
+
+[gcc]
+part = gcc-10.2
+min_version = 10.2
+
+[python]
+part = python3
+
+# Dependencies for the Cython+ test suite
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${numpy:egg}
+  coverage
+  pycodestyle
+
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/cython.git
+git-executable = ${git:location}/bin/git
+sparse-checkout = /.gitignore
+
+[cythonplus_env.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template =
+  inline:{% set path, python = os.path.split(python) -%}
+  {% if 'part' in gcc -%}
+  {%   set path = path + ':' + gcc.prefix + '/bin' -%}
+  {% endif -%}
+  export PATH={{ path }}$${PATH:+:$PATH}
+  export PYTHON={{ python }}
+  export PYTHONPATH={{ cythonplus_repository }}$${PYTHONPATH:+:$PYTHONPATH}
+  export PYTHONPATH={{ ':'.join(easy_install.working_set(eggs['eggs'].split(), [
+      eggs['develop-eggs-directory'],
+      eggs['eggs-directory'],
+    ]).entries) }}$${PYTHONPATH:+:$PYTHONPATH}
+  {# Set path to libintl needed for cython EmbedTest #}
+  export LD_RUN_PATH={{ gettext }}/lib$${LD_RUN_PATH:+:$LD_RUN_PATH}
+  export LIBRARY_PATH={{ gettext }}/lib$${LIBRARY_PATH:+:$LIBRARY_PATH}
+context =
+  section gcc gcc
+  section eggs eggs
+  key cythonplus_repository cythonplus-repository:location
+  key python python:executable
+  key gettext gettext:location
+  import os os
+  import easy_install zc.buildout.easy_install
+
+[versions]
+slapos.recipe.template = 4.4
+coverage = 4.5.1
+pycodestyle = 2.5.0

software/cythonplus-dev/buildout.hash.cfg

+[instance.cfg]
+filename = instance.cfg.in
+md5sum = a24384487467a07e827edab17a0b7206
+
+[runTestSuite.in]
+_update_hash_filename_ = runTestSuite.in
+md5sum = 21a8a202b14475707c414056ba393b3d

software/cythonplus-dev/instance.cfg.in

+[buildout]
+
+parts =
+  publish-env-path
+  runTestSuite
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[publish-env-path]
+recipe = slapos.cookbook:publish
+readme = Source the script to set up the environment.
+script = ${cythonplus_env.sh:rendered}
+repository = ${cythonplus-repository:repository}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = ${cythonplus-repository:location}
+git-executable = ${git:location}/bin/git
+shared = true
+
+[runTestSuite]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+template = ${runTestSuite.in:target}
+mode = 0755
+context =
+  key tmpdir directory:tmp
+  key slapparameter_dict slap-configuration:configuration
+  key cythonplus_repository cythonplus-repository:location
+  raw runTestSuite_interpreter ${runTestSuite_interpreter:bin-directory}/${runTestSuite_interpreter:interpreter}
+  raw cythonplus_env_sh ${cythonplus_env.sh:rendered}

software/cythonplus-dev/runTestSuite.in

+#!{{ runTestSuite_interpreter }}
+"""
+  Script to run the Cython test suite using Nexedi's test node framework.
+"""
+import argparse, json, os, subprocess, sys, traceback
+from time import gmtime, strftime, time
+from unittest import TextTestResult
+from erp5.util import taskdistribution, testsuite
+
+os.environ['TEMP'] = {{ repr(tmpdir) }}
+
+command = """. {{ cythonplus_env_sh }}
+make all test
+"""
+
+def parseTestStdOut(data):
+  """
+  Parse output of Cython testrunner script.
+  """
+  data = data.rsplit(b'\n' + TextTestResult.separator2.encode() + b'\n', 1)[1]
+  status_dict = {}
+  search = testsuite.TestSuite.RUN_RE.search(data)
+  if search:
+    groupdict = search.groupdict()
+    status_dict.update(duration=float(groupdict['seconds']),
+                       test_count=int(groupdict['all_tests']))
+  search = testsuite.TestSuite.STATUS_RE.search(data)
+  if search:
+    groupdict = search.groupdict()
+    status_dict.update(
+      error_count=int(groupdict['errors'] or 0),
+      failure_count=int(groupdict['failures'] or 0)
+                   +int(groupdict['unexpected_successes'] or 0),
+      skip_count=int(groupdict['skips'] or 0)
+                +int(groupdict['expected_failures'] or 0))
+  return status_dict
+
+class DummyTestResult:
+
+  class DummyTestResultLine:
+
+    def stop(self, **kw):
+      with open(self.name + '.json', 'w') as f:
+        json.dump(kw, f)
+
+  done = 0
+
+  def __init__(self, test_name_list):
+    self.test_name_list = test_name_list
+
+  def start(self):
+    test_result_line = self.DummyTestResultLine()
+    try:
+      test_result_line.name = self.test_name_list[self.done]
+    except IndexError:
+      return
+    self.done += 1
+    return test_result_line
+
+def main():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='ignored', type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+
+  args = parser.parse_args()
+
+  test_suite_title = args.test_suite_title or args.test_suite
+
+  test_name_list = 'cython',
+
+  if args.master_url:
+    tool = taskdistribution.TaskDistributor(portal_url = args.master_url)
+    test_result = tool.createTestResult(args.revision,
+                                        test_name_list,
+                                        args.test_node_title,
+                                        test_title=test_suite_title,
+                                        project_title=args.project_title)
+    if test_result is None:
+      return
+  else:
+    test_result = DummyTestResult(test_name_list)
+  # run NEO tests
+  while 1:
+    test_result_line = test_result.start()
+    if not test_result_line:
+      break
+
+    try:
+      with open(os.devnull) as stdin:
+        p = subprocess.Popen(command, shell=True, stdin=stdin,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                             cwd={{ repr(cythonplus_repository) }})
+    except Exception:
+      end = time()
+      stderr = traceback.format_exc()
+      status_dict = {}
+      sys.stderr.write(stderr)
+    else:
+      stdout, stderr = p.communicate()
+      end = time()
+      os.write(1, stdout)
+      os.write(2, stderr)
+      status_dict = parseTestStdOut(stderr)
+      if str is not bytes:
+        stdout = stdout.decode('utf-8')
+        stderr = stderr.decode('utf-8')
+      status_dict['stdout'] = stdout
+
+    # report status back to Nexedi ERP5
+    test_result_line.stop(
+      command=command,
+      date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      stderr=stderr,
+      **status_dict)
+
+if __name__ == "__main__":
+    main()

software/cythonplus-dev/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/cythonplus/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance.cfg
+
+[instance.cfg]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+
+[runTestSuite.in]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[runTestSuite_interpreter]
+recipe = zc.recipe.egg
+eggs = erp5.util
+interpreter = ${:_buildout_section_name_}
+
+[versions]
+slapos.recipe.template = 4.4

[Jérome Perrin]

You should take a look at nxdtest, and how we are now using it to replace runTestSuite.in, see 662b1a0b , 9a5cfb73 , 805374b4 or !862 (merged) maybe instead of writing this code we can use nxdtest here as well.

Since nxdtest@bca50060 nxdtest know how to parse unittest output. For this, maybe using a .nxdtest file with

TestCase('test', ['make', 'all', 'test'], summaryf=UnitTest.summary, cwd={{ repr(cythonplus_repository) }})

would be enough.

/cc @kirr