Commit b78f451b authored by Boxiang Sun's avatar Boxiang Sun

Port pyodide to slapos

parent 5e79e46e
[buildout]
# slaposcookbook_recipe_location = ${:parts-directory}/slaposcookbook
extends =
../curl/buildout.cfg
../git/buildout.cfg
../python3/buildout.cfg
../ccache/buildout.cfg
../nodejs/buildout.cfg
../wget/buildout.cfg
../tar/buildout.cfg
../libyaml/buildout.cfg
../python-PyYAML/buildout.cfg
../cmake/buildout.cfg
../bzip2/buildout.cfg
../file/buildout.cfg
../gcc/buildout.cfg
../gdbm/buildout.cfg
../gettext/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../libpng/buildout.cfg
../freetype/buildout.cfg
../pkgconfig/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
../f2c/buildout.cfg
parts =
pyodide
pyodide-script-test
nodejs
npm
python3.7.2-PyYAML
[node-less]
recipe = slapos.recipe.build:npm
packages = less
node = nodejs
environment =
PATH=${nodejs:location}/bin:%(PATH)s
[node-uglify-js]
recipe = slapos.recipe.build:npm
packages = uglify-js
node = nodejs
environment =
PATH=${nodejs:location}/bin:%(PATH)s
[pyodide-src]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/Daetalus/pyodide.git
location = ${buildout:parts-directory}/${:_buildout_section_name_}
branch = master
[pyodide-script]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
build_dir = ${buildout:parts-directory}/pyodide-script-test
part_dir = ${buildout:parts-directory}
git-executable = ${git:location}/bin/git
emsdk_set_env_script = ${:build_dir}/emsdk/emsdk/emsdk_set_env.sh
repository = https://lab.nexedi.com/Daetalus/pyodide.git
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig
PATH=${curl:location}/bin:${git:location}/bin:${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}:${python3.7.2:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin
CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LD_LIBRARY_PATH=${zlib:location}/lib:${libpng:location}/lib:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
CPATH=${f2c:location}/include:${zlib:location}/include:${xz-utils:location}/include:${readline:location}/include:${libexpat:location}/include:${libffi:location}/include:${ncurses:location}/include:${ncurses:location}/include/ncursesw:${bzip2:location}/include:${gdbm:location}/include:${openssl:location}/include:${sqlite3:location}/include:${gettext:location}/include
openssl_location = ${openssl:location}
gcc_bin_dir = ${gcc-8.2:location}/bin
python_bin_dir = ${python3.7.2:location}/bin
stop-on-error=true
script =
import subprocess, os, shutil
env = {
'PKG_CONFIG_PATH':self.options['PKG_CONFIG_PATH'],
'PATH':self.options['PATH'] + ':' + os.environ['PATH'],
'CPATH':self.options['CPATH'],
'CPPFLAGS':self.options['CPPFLAGS'],
'LDFLAGS':self.options['LDFLAGS'],
}
if not os.path.isdir(self.options['build_dir']):
command_list_0 = [{'cmd': self.options['git-executable'] + ' clone ' + self.options['repository'] + ' pyodide-script-test', 'cwd': self.options['part_dir']}]
# put the Popen in the loop to let the pipe.wait() work.
for command_0 in command_list_0:
pipe = subprocess.Popen(command_0['cmd'], cwd=command_0['cwd'], env=env, shell=True)
pipe.wait()
# else:
# shutil.rmtree(self.options['build_dir'], ignore_errors=True)
cpython_makefile_command = 'sed -ie "s@./configure --prefix@./configure --with-openssl=' + self.options['openssl_location'] + ' --prefix@" ' + self.options['build_dir'] + '/cpython/Makefile'
get_pip = "curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
install_pip = "python3 get-pip.py"
install_pytest = "pip install pytest selenium pytest-instafail pytest-timeout"
# Hack: create a gfortran symlink inside the python bin dir.
# We have to build the pyodide cpython along with SlapOS host gcc, not the custom gcc in the parts dir.
# Because if we use custom gcc to build the pyodide cpython, later we build pyodide python extension, the distutil going to
# expand the gcc which used by the pyodide cpython to full path instead plain `gcc`.
# Pyodide did some hacks to make distutil support crossing compile, which require use the plain `gcc` when building python extension.
# So we are not going to provide the custom gcc bin path in the PATH environment variable.
# But scipy needs gfortran. The workaround is just give a symlink in the cpython bin dir, which points to the custom gcc gfortran.
# Note: the parts/cpython is for run the pyodide building script. The pyodide/cpython is for compile to webassembly module which will loads to the browser.
create_gfortran_symlink = 'ln -s ' + self.options['gcc_bin_dir'] + '/gfortran ' + self.options['python_bin_dir'] + '/gfortran'
command_list = [
{'cmd': cpython_makefile_command, 'cwd': self.options['build_dir']},
{'cmd': get_pip, 'cwd': self.options['build_dir']},
{'cmd': install_pip, 'cwd': self.options['build_dir']},
{'cmd': install_pytest, 'cwd': self.options['build_dir']},
{'cmd': create_gfortran_symlink, 'cwd': self.options['build_dir']},
{'cmd': 'make -C emsdk', 'cwd': self.options['build_dir']},
{'cmd': './emsdk/emsdk/emsdk_env.sh', 'cwd': self.options['build_dir']},
{'cmd': 'make -C lz4', 'cwd': self.options['build_dir']},
{'cmd': 'make -C cpython', 'cwd': self.options['build_dir']},
{'cmd': 'make -C CLAPACK', 'cwd': self.options['build_dir']},
{'cmd': 'make -C packages', 'cwd': self.options['build_dir']},
]
for command in command_list:
pipe = subprocess.Popen(command['cmd'], cwd=command['cwd'], env=env, shell=True)
pipe.wait()
# We need to execute the command "./emsdk/emsdk/emsdk_env.sh" to add the emsdk related environment variables.
# Run "emsdk_env.sh" first, then read the content of "emsdk_set_env.sh" to get the environment variables
emsdk_env_content = open(self.options['emsdk_set_env_script'])
for content in emsdk_env_content:
# remove "export " prefix
content = content[7:]
key, value = content.split('=')
value = value.rstrip().strip('"')
try:
env[key] += ':' + value
except KeyError:
env[key] = value
env['CPATH'] += env['EMSCRIPTEN'] + '/system/include/'
command_list_2 = [
{'cmd': 'make build/pyodide.asm.js', 'cwd': self.options['build_dir']},
{'cmd': 'make -e', 'cwd': self.options['build_dir']},
]
for command_2 in command_list_2:
pipe = subprocess.Popen(command_2['cmd'], cwd=command_2['cwd'], env=env, shell=True)
pipe.wait()
[python3.7.2-PyYAML]
recipe = slapos.recipe.cmmi
python_bin = ${python3.7.2:location}/bin/python3.7
url = https://pyyaml.org/download/pyyaml/PyYAML-3.13.tar.gz
configure-command = true
make-binary = ${:python_bin} setup.py install
environment =
C_INCLUDE_PATH=${libyaml:location}/include
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[pyodide-emsdk]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/emsdk
configure-command= alias python=${python3.7.2:location}/bin/python3 && make clean
make-options = -e
environment =
PATH=${f2c:location}/bin:${python3.7.2:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
PYTHON_EXECUTABLE=${python3.7.2:location}/bin/python3.7
make-targets =
# Build pyodide cpython with custom configuration. Otherwise we will get a cpython without ssl support
# The pyodide cpython **must** been build **AFTER** emsdk
[pyodide-cpython]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}/cpython
configure-command = sed -ie "s@./configure --prefix@./configure --with-openssl=${openssl:location} --prefix@" ${pyodide-src:location}/cpython/Makefile && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
environment =
PATH=${xz-utils:location}/bin:${gcc-8.2:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
make-targets =
[pyodide-packages]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = alias python3=${pyodide-cpython:path}/build/3.7.0/host/bin/python3 && source ${pyodide-emsdk:path}/emsdk/emsdk_env.sh
make-options = -e
make-targets =
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${gcc-8.2:location}/bin:${f2c:location}/bin:${pkgconfig:location}/bin:${pyodide-cpython:path}/build/3.7.0/host/bin:${nodejs:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:%(PATH)s
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64
[pyodide]
recipe = slapos.recipe.cmmi
path = ${pyodide-src:location}
configure-command = make -e
make-binary =
make-targets = cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_}
# configure-command = echo ${python3.7.2-PyYAML:location} && echo ${pyodide-cpython:path} && echo ${pyodide-emsdk:path} && echo ${node-uglify-js:location} && echo ${node-less:location}# ${pyodide-packages:path}
configure-command = echo ${python3.7.2-PyYAML:location} && echo ${pyodide-script:location}
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${pkgconfig:location}/bin:${freetype:location}/bin:${nodejs:location}/bin/:${f2c:location}/bin:${python3.7.2:location}/bin:${cmake:location}/bin:${ccache:location}/bin:${gcc-8.2:location}/bin:${node-less:location}/node_modules/.bin:${node-uglify-js:location}/node_modules/.bin:${xz-utils:location}/bin:%(PATH)s
CC=${gcc-8.2:location}/bin/gcc
CXX=${gcc-8.2:location}/bin/g++
CPPFLAGS=-I${f2c:location}/include -I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LD_LIBRARY_PATH=${libpng:location}/lib:${gcc-8.2:location}/lib:${gcc-8.2:location}/lib64:${openssl:location}/lib:${readline:location}/lib:${libexpat:location}/lib:${xz-utils:location}/lib:${sqlite3:location}/lib:${gdbm:location}/lib:${bzip2:location}/lib:${ncurses:location}/lib:${libffi:location}/lib
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
PYTHON_EXECUTABLE=${python3.7.2:location}/bin/python3.7
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment