Commit e6b930c0 authored by Boxiang Sun's avatar Boxiang Sun

Port pyodide to slapos

Build pyodide on SlapOS
Add test for pyodide compiling
parent f12cb16b
[buildout]
# slaposcookbook_recipe_location = ${:parts-directory}/slaposcookbook
extends =
../curl/buildout.cfg
../git/buildout.cfg
../python3/buildout.cfg
../ccache/buildout.cfg
../nodejs/buildout.cfg
../wget/buildout.cfg
../tar/buildout.cfg
../libyaml/buildout.cfg
../python-PyYAML/buildout.cfg
../cmake/buildout.cfg
../bzip2/buildout.cfg
../file/buildout.cfg
../gcc/buildout.cfg
../gdbm/buildout.cfg
../gettext/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../libpng/buildout.cfg
../freetype/buildout.cfg
../pkgconfig/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
../f2c/buildout.cfg
parts =
pyodide
pyodide-script-test
nodejs
npm
python3.7-PyYAML
[node-less]
recipe = slapos.recipe.build:npm
packages = less
node = nodejs
environment =
PATH=${nodejs:location}/bin:%(PATH)s
[node-uglify-js]
recipe = slapos.recipe.build:npm
packages = uglify-js
node = nodejs
environment =
PATH=${nodejs:location}/bin:%(PATH)s
[pyodide-src]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/Daetalus/pyodide.git
location = ${buildout:parts-directory}/${:_buildout_section_name_}
branch = master
[pyodide-script]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
build_dir = ${buildout:parts-directory}/pyodide-script-test
part_dir = ${buildout:parts-directory}
git-executable = ${git:location}/bin/git
emsdk_set_env_script = ${:build_dir}/emsdk/emsdk/emsdk_set_env.sh
repository = https://lab.nexedi.com/Daetalus/pyodide.git
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig
PATH=${curl:location}/bin:${git:location}/bin:${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${libffi:location}/include -I${gettext:location}/include
LD_LIBRARY_PATH=${zlib:location}/lib:${libpng:location}/lib:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
LDFLAGS= -L${libffi:location}/lib -L${zlib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
CPATH=${f2c:location}/include:${zlib:location}/include:${xz-utils:location}/include:${readline:location}/include:${libexpat:location}/include:${libffi:location}/include:${ncurses:location}/include:${ncurses:location}/include/ncursesw:${bzip2:location}/include:${gdbm:location}/include:${openssl:location}/include:${sqlite3:location}/include:${gettext:location}/include
openssl_location = ${openssl:location}
gcc_bin_dir = ${gcc-8.2:location}/bin
python_bin_dir = ${python3.7:location}/bin
md5sum = 47ec6a091e503349d948760dc03bed51
pip_script = ${:_profile_base_location_}/get-pip.py
script =
import subprocess, os, shutil
env = {
'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
'CPATH':self.options['CPATH'],
'CPPFLAGS':self.options['CPPFLAGS'],
'LDFLAGS':self.options['LDFLAGS'],
}
if not os.path.isdir(self.options['build_dir']):
command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
# put the Popen in the loop to let the pipe.wait() work.
for command_0 in command_list_0:
pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
pipe.wait()
# else:
# shutil.rmtree(self.options['build_dir'], ignore_errors=True)
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
pip_script = self.options['pip_script']
install_pip = "python3 " + pip_script
install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
# Hack: create a gfortran symlink inside the python bin dir.
# We have to build the pyodide cpython along with SlapOS host gcc, not the custom gcc in the parts dir.
# Because if we use custom gcc to build the pyodide cpython, later we build pyodide python extension, the distutil going to
# expand the gcc which used by the pyodide cpython to full path instead plain `gcc`.
# Pyodide did some hacks to make distutil support crossing compile, which require use the plain `gcc` when building python extension.
# So we are not going to provide the custom gcc bin path in the PATH environment variable.
# But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
# Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
command_list = [
{'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
{'cmd': install_pip, 'cwd': self.options['build_dir']},
{'cmd': install_pytest, 'cwd': self.options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
{'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
]
for command in command_list:
pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
pipe.wait()
# We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
# Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
emsdk_env_content = open(self.options['emsdk_set_env_script'])
for content in emsdk_env_content:
# remove "export " prefix
content = content[7:]
key, value = content.split('=')
value = value.rstrip().strip('"')
try:
env[key] += ':' + value
except KeyError:
env[key] = value
env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
command_list_2 = [
{'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
{'cmd': 'make -e', 'cwd': self.options['build_dir']},
]
for command_2 in command_list_2:
pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
pipe.wait()
[python3.7-PyYAML]
recipe = slapos.recipe.cmmi
python_bin = ${python3.7:location}/bin/python3.7
url = https://pyyaml.org/download/pyyaml/PyYAML-3.13.tar.gz
configure-command = true
make-binary = ${:python_bin} setup.py install
environment =
C_INCLUDE_PATH=${libyaml:location}/include
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pyodide-emsdk]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/emsdk
configure-command= alias python=${python3.7:location}/bin/python3 && make clean
make-options = -e
environment =
PATH=${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7
make-targets =
# Build pyodide cpython with custom configuration. Otherwise we will get a cpython without ssl support
# The pyodide cpython **must** been build **AFTER** emsdk
[pyodide-cpython]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/cpython
configure-command = sed -ie "s@./configure --prefix@./configure --with-openssl=${openssl:location} --prefix@" ${pyodide-src:location}/cpython/Makefile && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
environment =
PATH=${xz-utils:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
make-targets =
[pyodide-packages]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = alias python3=${pyodide-cpython:path}/build/3.7.0/host/bin/python3 && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
make-targets =
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${gcc-8.2:location}/bin:${f2c:location}/bin:${pkgconfig:location}/bin:${pyodide-cpython:path}/build/3.7.0/host/bin:${nodejs:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:%(PATH)s
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
[pyodide]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = make -e
make-binary =
make-targets = cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_}
# configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-cpython:path} && echo ${pyodide-emsdk:path} && echo ${node-uglify-js:location} && echo ${node-less:location}# ${pyodide-packages:path}
configure-command = echo ${python3.7-PyYAML:location} && echo ${pyodide-script:location}
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}/bin:${python3.7:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
CPPFLAGS=-I${f2c:location}/include -I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
PYTHON_EXECUTABLE=${python3.7:location}/bin/python3.7
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance]
filename = instance.cfg.in
md5sum = e4e96f0f08b8de6807a24f43fce34b40
[template-runTestSuite]
filename = runTestSuite.in
md5sum = 28fb6c00c7568a6349e9ff1a9b09fc53
\ No newline at end of file
[buildout]
parts =
runTestSuite-instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = false
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
#################################
# Test runner
#################################
[runTestSuite-instance]
recipe = slapos.recipe.template
url = ${template-runTestSuite:output}
output = $${directory:bin}/runTestSuite
buildout-directory = $${buildout:directory}
mode = 0700
\ No newline at end of file
#!${buildout:directory}/bin/${eggs:interpreter}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
"""
Script to run Cython test suite using Nexedi's test node framework.
"""
import argparse, os, re, shutil, subprocess, sys, traceback
from erp5.util import taskdistribution
from time import gmtime, strftime
from subprocess import check_output
import importlib
import datetime
def main():
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', help='ignored', type=int)
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
parser.add_argument('--frontend_url',
help='The url of frontend of the test suite')
args = parser.parse_args()
is_browser_running = False
try:
test_suite_title = args.test_suite_title or args.test_suite
test_suite = args.test_suite
revision = args.revision
test_line_dict = {}
date = strftime("%Y/%m/%d %H:%M:%S", gmtime())
##########################
# Run all tests
##########################
path_var = os.pathsep.join([os.environ.get('PATH', os.defpath), '${python3.7:location}/bin', '${geckodriver:location}', '${firefox-wrapper:location}'])
env = dict(os.environ, PATH=path_var)
test_list = ['test_common', ]
failed = 0
result_string = None
duration = 0
test_line_dict = {}
for test_name in test_list:
failed = 0
result_string = None
duration = 0
try:
dir = '${pyodide:location}' + '-script-test'
print(os.path.isdir(dir))
print(os.path.isdir('${pyodide:location}'))
print(os.path.isdir(dir + '/test'))
print(path_var)
print(env)
result_string = check_output(['pytest', '--timeout=5', './test/test_common.py'],
cwd=dir,
env=env) # temporary solution for not block by xfailed tests too long
except OSError, e:
print("We get an OSError here")
result_string = str(e.strerror)
except Exception, e: # except CalledProcessError, e:
print("We get a generic error:")
print(e)
result_string = str(e.output)
print('${pyodide:location}')
print(result_string)
test_result_content = result_string.split('\n')
print('---------------------')
print('---------------------')
print('---------------------')
test_result_statistic_line = test_result_content[-2]
print(test_result_statistic_line)
# use magic code, something like
# === 21 passed, 1 warnings, 42 error in 130.43 seconds ===
test_result_statistic = test_result_statistic_line.split(' ')
print(test_result_statistic)
duration = test_result_statistic[-3]
failed = int(test_result_statistic[5])
stdout = result_string
test_line_dict['%s: %s' % ('Pyodide test', test_name)] = {
'test_count': int(test_result_statistic[1]),
'error_count': 0,
'failure_count': failed,
'skip_count': 0,
'duration': float(duration),
'command': '',
'stdout': stdout,
'stderr': '',
'html_test_result': '',
}
# Send results
tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(),
node_title = args.test_node_title,
test_title = test_suite_title,
project_title = args.project_title)
if test_result is None or not hasattr(args, 'master_url'):
return
# report test results
while 1:
test_result_line = test_result.start()
if not test_result_line:
print 'No test result anymore.'
break
print 'Submitting: "%s"' % test_result_line.name
print test_line_dict
# report status back to Nexedi ERP5
test_result_line.stop(**test_line_dict[test_result_line.name])
except:
# Catch any exception here, to warn user instead of being silent,
# by generating fake error result
print traceback.format_exc()
result = dict(status_code=-1,
command='pytest test_common.py',
stderr=traceback.format_exc(),
stdout='')
# XXX: inform test node master of error
raise EnvironmentError(result)
if __name__ == "__main__":
main()
\ No newline at end of file
[buildout]
extends =
../../stack/slapos.cfg
../../component/git/buildout.cfg
../../component/pyodide/buildout.cfg
../../component/firefox/buildout.cfg
../../component/chromium/buildout.cfg
../../component/chromedriver/buildout.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
git
instance
chromedriver
chromium
pyodide
template-runTestSuite
[eggs]
recipe = zc.recipe.egg
eggs =
erp5.util
interpreter = pythonwitheggs
[macro-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[instance]
<= macro-template
output = ${buildout:directory}/instance.cfg
[template-runTestSuite]
<= macro-template
output = ${buildout:directory}/runTestSuite.in
[versions]
slapos.recipe.template = 4.3
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment