Port pyodide to slapos

Build pyodide on SlapOS
Add test for pyodide compiling

component/pyodide/buildout.cfg

+[buildout]
+# slaposcookbook_recipe_location = ${:parts-directory}/slaposcookbook
+
+extends =
+  ../curl/buildout.cfg
+  ../git/buildout.cfg
+  ../python3/buildout.cfg
+  ../ccache/buildout.cfg
+  ../nodejs/buildout.cfg
+  ../wget/buildout.cfg
+  ../tar/buildout.cfg
+  ../libyaml/buildout.cfg
+  ../python-PyYAML/buildout.cfg
+  ../cmake/buildout.cfg
+  ../bzip2/buildout.cfg
+  ../file/buildout.cfg
+  ../gcc/buildout.cfg
+  ../gdbm/buildout.cfg
+  ../gettext/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../libffi/buildout.cfg
+  ../libpng/buildout.cfg
+  ../freetype/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../ncurses/buildout.cfg
+  ../openssl/buildout.cfg
+  ../patch/buildout.cfg
+  ../readline/buildout.cfg
+  ../sqlite3/buildout.cfg
+  ../xz-utils/buildout.cfg
+  ../zlib/buildout.cfg
+  ../f2c/buildout.cfg
+
+parts =
+  pyodide
+  pyodide-script-test
+  nodejs
+  npm
+  python3.7-PyYAML
+
+[node-less]
+recipe = slapos.recipe.build:npm
+packages = less
+node = nodejs
+environment =
+  PATH=${nodejs:location}/bin:%(PATH)s
+
+[node-uglify-js]
+recipe = slapos.recipe.build:npm
+packages = uglify-js
+node = nodejs
+environment =
+  PATH=${nodejs:location}/bin:%(PATH)s
+
+[pyodide-src]
+recipe = slapos.recipe.build:gitclone
+
+git-executable = ${git:location}/bin/git
+repository = https://lab.nexedi.com/Daetalus/pyodide.git
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+branch = master
+
+[pyodide-script]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+build_dir =  ${buildout:parts-directory}/pyodide-script-test
+part_dir = ${buildout:parts-directory}
+git-executable = ${git:location}/bin/git
+emsdk_set_env_script = ${:build_dir}/emsdk/emsdk/emsdk_set_env.sh
+repository = https://lab.nexedi.com/Daetalus/pyodide.git
+PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig
+PATH=${curl:location}/bin:${git:location}/bin:${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
+CPPFLAGS=-I${zlib:location}/include -I${libffi:location}/include -I${gettext:location}/include
+LD_LIBRARY_PATH=${zlib:location}/lib:${libpng:location}/lib:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
+LDFLAGS= -L${libffi:location}/lib -L${zlib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+CPATH=${f2c:location}/include:${zlib:location}/include:${xz-utils:location}/include:${readline:location}/include:${libexpat:location}/include:${libffi:location}/include:${ncurses:location}/include:${ncurses:location}/include/ncursesw:${bzip2:location}/include:${gdbm:location}/include:${openssl:location}/include:${sqlite3:location}/include:${gettext:location}/include
+openssl_location = ${openssl:location}
+gcc_bin_dir = ${gcc-8.2:location}/bin
+python_bin_dir = ${python3.7:location}/bin
+md5sum = 47ec6a091e503349d948760dc03bed51
+pip_script = ${:_profile_base_location_}/get-pip.py
+script =
+  import subprocess, os, shutil
+
+  env = {
+    'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
+    'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
+    'CPATH':self.options['CPATH'],
+    'CPPFLAGS':self.options['CPPFLAGS'],
+    'LDFLAGS':self.options['LDFLAGS'],
+  }
+
+  if not os.path.isdir(self.options['build_dir']):
+    command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
+    # put the Popen in the loop to let the pipe.wait() work.
+    for command_0 in command_list_0:
+      pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
+      pipe.wait()
+  # else:
+    # shutil.rmtree(self.options['build_dir'], ignore_errors=True)
+
+  cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
+  pip_script = self.options['pip_script']
+  install_pip = "python3 " + pip_script
+  install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
+  # Hack: create a gfortran symlink inside the python bin dir.
+  # We have to build the pyodide cpython along with SlapOS host gcc, not the custom gcc in the parts dir.
+  # Because if we use custom gcc to build the pyodide cpython, later we build pyodide python extension, the distutil going to
+  # expand the gcc which used by the pyodide cpython to full path instead plain `gcc`.
+  # Pyodide did some hacks to make distutil support crossing compile, which require use the plain `gcc` when building python extension.
+  # So we are not going to provide the custom gcc bin path in the PATH environment variable.
+  # But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
+  # Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
+  create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
+
+  command_list = [
+    {'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
+    {'cmd': install_pip, 'cwd': self.options['build_dir']},
+    {'cmd': install_pytest, 'cwd': self.options['build_dir']},
+    {'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
+    {'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
+    {'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
+    {'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
+    {'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
+    {'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
+    {'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
+  ]
+
+  for command in command_list:
+    pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
+    pipe.wait()
+  
+  # We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
+  # Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
+  emsdk_env_content = open(self.options['emsdk_set_env_script'])
+  for content in emsdk_env_content:
+    # remove "export " prefix
+    content = content[7:]
+    key, value = content.split('=')
+    value = value.rstrip().strip('"')
+    try:
+      env[key] += ':' + value
+    except KeyError:
+      env[key] = value
+  env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
+
+  command_list_2 = [
+    {'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
+    {'cmd': 'make -e', 'cwd': self.options['build_dir']},
+  ]
+  
+  for command_2 in command_list_2:
+    pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
+    pipe.wait()
+
+[python3.7-PyYAML]
+recipe = slapos.recipe.cmmi
+python_bin = ${python3.7:location}/bin/python3.7
+url = https://pyyaml.org/download/pyyaml/PyYAML-3.13.tar.gz
+configure-command = true
+make-binary = ${:python_bin} setup.py install
+environment =
+  C_INCLUDE_PATH=${libyaml:location}/include
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[pyodide-emsdk]
+recipe = slapos.recipe.cmmi
+path = ${pyodide-src:location}/emsdk
+configure-command= alias python=${python3.7:location}/bin/python3 && make clean
+make-options = -e
+environment =
+  PATH=${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
+  CC=${gcc-8.2:location}/bin/gcc
+  CXX=${gcc-8.2:location}/bin/g++
+  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
+  PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7
+make-targets =
+
+# Build pyodide cpython with custom configuration. Otherwise we will get a cpython without ssl support
+# The pyodide cpython **must** been build **AFTER** emsdk
+[pyodide-cpython]
+recipe = slapos.recipe.cmmi
+path = ${pyodide-src:location}/cpython
+configure-command = sed -ie "s@./configure --prefix@./configure --with-openssl=${openssl:location} --prefix@" ${pyodide-src:location}/cpython/Makefile && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
+make-options = -e
+environment =
+  PATH=${xz-utils:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
+  CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
+  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+  CC=${gcc-8.2:location}/bin/gcc
+  CXX=${gcc-8.2:location}/bin/g++
+  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
+make-targets =
+
+[pyodide-packages]
+recipe = slapos.recipe.cmmi
+path = ${pyodide-src:location}
+configure-command = alias python3=${pyodide-cpython:path}/build/3.7.0/host/bin/python3 && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
+make-options = -e
+make-targets =
+environment =
+  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
+  PATH=${gcc-8.2:location}/bin:${f2c:location}/bin:${pkgconfig:location}/bin:${pyodide-cpython:path}/build/3.7.0/host/bin:${nodejs:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:%(PATH)s
+  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
+
+[pyodide]
+recipe = slapos.recipe.cmmi
+path = ${pyodide-src:location}
+configure-command = make -e
+make-binary =
+make-targets = cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_} 
+# configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-cpython:path} && echo ${pyodide-emsdk:path} && echo ${node-uglify-js:location} && echo ${node-less:location}# ${pyodide-packages:path}
+configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-script:location}
+environment =
+  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
+  PATH=${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
+  CC=${gcc-8.2:location}/bin/gcc
+  CXX=${gcc-8.2:location}/bin/g++
+  CPPFLAGS=-I${f2c:location}/include -I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
+  LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+  PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7
\ No newline at end of file

software/pyodide/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance]
+filename = instance.cfg.in
+md5sum = e4e96f0f08b8de6807a24f43fce34b40
+
+[template-runTestSuite]
+filename = runTestSuite.in
+md5sum = 28fb6c00c7568a6349e9ff1a9b09fc53
\ No newline at end of file

software/pyodide/instance.cfg.in

+[buildout]
+parts =
+  runTestSuite-instance
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = false
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+#################################
+# Test runner
+#################################
+[runTestSuite-instance]
+recipe = slapos.recipe.template
+url = ${template-runTestSuite:output}
+output = $${directory:bin}/runTestSuite
+buildout-directory = $${buildout:directory}
+mode = 0700
\ No newline at end of file

software/pyodide/runTestSuite.in

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+"""
+  Script to run Cython test suite using Nexedi's test node framework.
+"""
+import argparse, os, re, shutil, subprocess, sys, traceback
+from erp5.util import taskdistribution
+from time import gmtime, strftime
+from subprocess import check_output
+import importlib
+import datetime
+def main():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='ignored', type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+  parser.add_argument('--frontend_url',
+                      help='The url of frontend of the test suite')
+  args = parser.parse_args()
+  is_browser_running = False
+  try:
+    test_suite_title = args.test_suite_title or args.test_suite
+    test_suite = args.test_suite
+    revision = args.revision
+    test_line_dict = {}
+    date = strftime("%Y/%m/%d %H:%M:%S", gmtime())
+    ##########################
+    # Run all tests
+    ##########################
+    path_var = os.pathsep.join([os.environ.get('PATH', os.defpath), '${python3.7:location}/bin', '${geckodriver:location}', '${firefox-wrapper:location}'])
+    env = dict(os.environ, PATH=path_var)
+    test_list = ['test_common', ]
+    failed = 0
+    result_string = None
+    duration = 0
+    test_line_dict = {}
+    for test_name in test_list:
+      failed = 0
+      result_string = None
+      duration = 0
+
+      try:
+        dir = '${pyodide:location}' + '-script-test'
+        print(os.path.isdir(dir))
+        print(os.path.isdir('${pyodide:location}'))
+        print(os.path.isdir(dir + '/test'))
+        print(path_var)
+        print(env)
+
+        result_string = check_output(['pytest', '--timeout=5', './test/test_common.py'],
+                                   cwd=dir,
+                                   env=env) # temporary solution for not block by xfailed tests too long
+      except OSError, e:
+        print("We get an OSError here")
+        result_string = str(e.strerror)
+      except Exception, e: # except CalledProcessError, e:
+        print("We get a generic error:")
+        print(e)
+        result_string = str(e.output)
+      print('${pyodide:location}')
+      print(result_string)
+      test_result_content = result_string.split('\n')
+      print('---------------------')
+      print('---------------------')
+      print('---------------------')
+      test_result_statistic_line = test_result_content[-2]
+      print(test_result_statistic_line)
+      # use magic code, something like
+      # === 21 passed, 1 warnings, 42 error in 130.43 seconds ===
+      test_result_statistic = test_result_statistic_line.split(' ')
+      print(test_result_statistic)
+      duration = test_result_statistic[-3]
+      failed = int(test_result_statistic[5])
+      stdout = result_string
+      test_line_dict['%s: %s' % ('Pyodide test', test_name)] = {
+          'test_count': int(test_result_statistic[1]),
+          'error_count': 0,
+          'failure_count': failed,
+          'skip_count': 0,
+          'duration': float(duration),
+          'command': '',
+          'stdout': stdout,
+          'stderr': '',
+          'html_test_result': '',
+        }
+    # Send results
+    tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
+    test_result = tool.createTestResult(revision = revision,
+                                        test_name_list = test_line_dict.keys(),
+                                        node_title = args.test_node_title,
+                                        test_title = test_suite_title,
+                                        project_title = args.project_title)
+    if test_result is None or not hasattr(args, 'master_url'):
+      return
+    # report test results
+    while 1:
+      test_result_line = test_result.start()
+      if not test_result_line:
+        print 'No test result anymore.'
+        break
+      print 'Submitting: "%s"' % test_result_line.name
+      print test_line_dict
+      # report status back to Nexedi ERP5
+      test_result_line.stop(**test_line_dict[test_result_line.name])
+  except:
+    # Catch any exception here, to warn user instead of being silent,
+    # by generating fake error result
+    print traceback.format_exc()
+    result = dict(status_code=-1,
+                  command='pytest test_common.py',
+                  stderr=traceback.format_exc(),
+                  stdout='')
+    # XXX: inform test node master of error
+    raise EnvironmentError(result)
+if __name__ == "__main__":
+    main()
\ No newline at end of file

software/pyodide/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/git/buildout.cfg
+  ../../component/pyodide/buildout.cfg
+  ../../component/firefox/buildout.cfg
+  ../../component/chromium/buildout.cfg
+  ../../component/chromedriver/buildout.cfg
+  ./buildout.hash.cfg
+parts =
+  slapos-cookbook
+  git
+  instance
+  chromedriver
+  chromium
+  pyodide
+  template-runTestSuite
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  erp5.util
+interpreter = pythonwitheggs
+
+[macro-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[instance]
+<= macro-template
+output = ${buildout:directory}/instance.cfg
+
+[template-runTestSuite]
+<= macro-template
+output = ${buildout:directory}/runTestSuite.in
+
+[versions]
+slapos.recipe.template = 4.3
\ No newline at end of file

[Julien Muchembled]

Why such a big third-party file component/pyodide/get-pip.py in this repository ? Why isn't is downloaded with a download recipe ?

/cc @romain

[Julien Muchembled]

/cc @kazuhiko

[Romain Courteaud]

@Daetalus please use a fixed identical version from https://github.com/pypa/get-pip