Port pyodide to slapos

Build pyodide on SlapOS
Add test for pyodide compiling

software/pyodide/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance]
+filename = instance.cfg.in
+md5sum = e4e96f0f08b8de6807a24f43fce34b40
+
+[template-runTestSuite]
+filename = runTestSuite.in
+md5sum = 28fb6c00c7568a6349e9ff1a9b09fc53
\ No newline at end of file

software/pyodide/instance.cfg.in

+[buildout]
+parts =
+  runTestSuite-instance
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = false
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+#################################
+# Test runner
+#################################
+[runTestSuite-instance]
+recipe = slapos.recipe.template
+url = ${template-runTestSuite:output}
+output = $${directory:bin}/runTestSuite
+buildout-directory = $${buildout:directory}
+mode = 0700
\ No newline at end of file

software/pyodide/runTestSuite.in

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+"""
+  Script to run Cython test suite using Nexedi's test node framework.
+"""
+import argparse, os, re, shutil, subprocess, sys, traceback
+from erp5.util import taskdistribution
+from time import gmtime, strftime
+from subprocess import check_output
+import importlib
+import datetime
+def main():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='ignored', type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+  parser.add_argument('--frontend_url',
+                      help='The url of frontend of the test suite')
+  args = parser.parse_args()
+  is_browser_running = False
+  try:
+    test_suite_title = args.test_suite_title or args.test_suite
+    test_suite = args.test_suite
+    revision = args.revision
+    test_line_dict = {}
+    date = strftime("%Y/%m/%d %H:%M:%S", gmtime())
+    ##########################
+    # Run all tests
+    ##########################
+    path_var = os.pathsep.join([os.environ.get('PATH', os.defpath), '${python3.7:location}/bin', '${geckodriver:location}', '${firefox-wrapper:location}'])
+    env = dict(os.environ, PATH=path_var)
+    test_list = ['test_common', ]
+    failed = 0
+    result_string = None
+    duration = 0
+    test_line_dict = {}
+    for test_name in test_list:
+      failed = 0
+      result_string = None
+      duration = 0
+
+      try:
+        dir = '${pyodide:location}' + '-script-test'
+        print(os.path.isdir(dir))
+        print(os.path.isdir('${pyodide:location}'))
+        print(os.path.isdir(dir + '/test'))
+        print(path_var)
+        print(env)
+
+        result_string = check_output(['pytest', '--timeout=5', './test/test_common.py'],
+                                   cwd=dir,
+                                   env=env) # temporary solution for not block by xfailed tests too long
+      except OSError, e:
+        print("We get an OSError here")
+        result_string = str(e.strerror)
+      except Exception, e: # except CalledProcessError, e:
+        print("We get a generic error:")
+        print(e)
+        result_string = str(e.output)
+      print('${pyodide:location}')
+      print(result_string)
+      test_result_content = result_string.split('\n')
+      print('---------------------')
+      print('---------------------')
+      print('---------------------')
+      test_result_statistic_line = test_result_content[-2]
+      print(test_result_statistic_line)
+      # use magic code, something like
+      # === 21 passed, 1 warnings, 42 error in 130.43 seconds ===
+      test_result_statistic = test_result_statistic_line.split(' ')
+      print(test_result_statistic)
+      duration = test_result_statistic[-3]
+      failed = int(test_result_statistic[5])
+      stdout = result_string
+      test_line_dict['%s: %s' % ('Pyodide test', test_name)] = {
+          'test_count': int(test_result_statistic[1]),
+          'error_count': 0,
+          'failure_count': failed,
+          'skip_count': 0,
+          'duration': float(duration),
+          'command': '',
+          'stdout': stdout,
+          'stderr': '',
+          'html_test_result': '',
+        }
+    # Send results
+    tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
+    test_result = tool.createTestResult(revision = revision,
+                                        test_name_list = test_line_dict.keys(),
+                                        node_title = args.test_node_title,
+                                        test_title = test_suite_title,
+                                        project_title = args.project_title)
+    if test_result is None or not hasattr(args, 'master_url'):
+      return
+    # report test results
+    while 1:
+      test_result_line = test_result.start()
+      if not test_result_line:
+        print 'No test result anymore.'
+        break
+      print 'Submitting: "%s"' % test_result_line.name
+      print test_line_dict
+      # report status back to Nexedi ERP5
+      test_result_line.stop(**test_line_dict[test_result_line.name])
+  except:
+    # Catch any exception here, to warn user instead of being silent,
+    # by generating fake error result
+    print traceback.format_exc()
+    result = dict(status_code=-1,
+                  command='pytest test_common.py',
+                  stderr=traceback.format_exc(),
+                  stdout='')
+    # XXX: inform test node master of error
+    raise EnvironmentError(result)
+if __name__ == "__main__":
+    main()
\ No newline at end of file

software/pyodide/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/git/buildout.cfg
+  ../../component/pyodide/buildout.cfg
+  ../../component/firefox/buildout.cfg
+  ../../component/chromium/buildout.cfg
+  ../../component/chromedriver/buildout.cfg
+  ./buildout.hash.cfg
+parts =
+  slapos-cookbook
+  git
+  instance
+  chromedriver
+  chromium
+  pyodide
+  template-runTestSuite
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  erp5.util
+interpreter = pythonwitheggs
+
+[macro-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[instance]
+<= macro-template
+output = ${buildout:directory}/instance.cfg
+
+[template-runTestSuite]
+<= macro-template
+output = ${buildout:directory}/runTestSuite.in
+
+[versions]
+slapos.recipe.template = 4.3
\ No newline at end of file

[Julien Muchembled]

Why such a big third-party file component/pyodide/get-pip.py in this repository ? Why isn't is downloaded with a download recipe ?

/cc @romain

[Julien Muchembled]

/cc @kazuhiko

[Romain Courteaud]

@Daetalus please use a fixed identical version from https://github.com/pypa/get-pip