Skip to content

slapos
slapos
Commit 0a3fd404 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

slapos-master|wendelin: Add Python3 Software Release version 

  This is a PoC to compile wendelin and slapos-master with python3, and
  specially to run the tests.
parent 5cc35a9d
Show whitespace changes
Inline Side-by-side
Showing with 207 additions and 147 deletions
software/slapos-master/base.cfg 0 → 100644
View file @ 0a3fd404
[erp5-defaults]
wcfs-enable-default = true
[local-bt5-repository]
# Same as bt5-repository, but only local repository.
# Used to generate bt5lists.
list += ${vifib:location}/master/bt5
[default-bt5]
list =
erp5_full_text_mroonga_catalog
slapos_configurator
[erp5_repository_list]
repository_id_list += vifib/master
[erp5]
branch = erp5-vifib
[vifib]
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos-bin.git
branch = master
[eggs]
eggs +=
slapos.core
dummy +=
${vifib:location}
${slapos-bin:location}
extra-paths +=
${vifib:location}/master
${slapos-bin:location}
### Overwrite recipes to introduce customized changes
[download-base-part]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
[template-erp5]
< = download-base-part
filename = instance-erp5.cfg.in
[template-balancer]
< = download-base-part
filename = instance-balancer.cfg.in
[template-apache-backend-conf]
url = ${:_profile_base_location_}/${:filename}
[template-haproxy-cfg]
url = ${:_profile_base_location_}/${:filename}
software/slapos-master/buildout.hash.cfg
View file @ 0a3fd404
......@@ -14,11 +14,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = cf5fbee0b5e104015eb9eac53ba2d241
md5sum = 9f1245e834fb0861db59e7c5ec2d8805
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 8af2ed33ef2a57cf5c5df4dd5d834d69
md5sum = 87516dad789d74456d120866089d7efd
[template-apache-backend-conf]
filename = apache-backend.conf.in
......@@ -26,4 +26,4 @@ md5sum = aa11283ca8b80a91b508732cd3443f7b
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = fec6a312e4ef84b02837742992aaf495
md5sum = 3848f3b78811d72b7633973fad9c54e9
software/slapos-master/haproxy.cfg.in
View file @ 0a3fd404
......@@ -32,7 +32,7 @@ defaults
# to render a page
option httpclose
{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
{% for name, (port, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
listen {{ name }}
bind {{ parameter_dict['ip'] }}:{{ port }}
http-request set-header X-Balancer-Current-Cookie SERVERID
......
software/slapos-master/instance-balancer.cfg.in
View file @ 0a3fd404
......@@ -39,9 +39,9 @@ recipe = slapos.recipe.template:jinja2
{% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
{% set zope_family_address_list = [] -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% set has_webdav = [] -%}
......@@ -154,7 +154,7 @@ cert = ${apache-conf-ssl:cert}
{%- endif %}
[apache-conf-parameter-dict]
backend-list = {{ dumps(apache_dict.values()) }}
backend-list = {{ dumps(list(apache_dict.values())) }}
zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid
......@@ -207,7 +207,7 @@ input = inline:
promise = check_socket_listening
name = apache.py
config-host = {{ ipv4 }}
config-port = {{ apache_dict.values()[0][0] }}
config-port = {{ list(apache_dict.values())[0][0] }}
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
......
software/slapos-master/instance-erp5.cfg.in
View file @ 0a3fd404
......@@ -11,6 +11,10 @@
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set with_max_rlimit_nofile_enable = slapparameter_dict.get(
'with-max-rlimit-nofile',
with_max_rlimit_nofile_enable_default.lower() in ('true', 'yes'))
%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -32,7 +36,7 @@
{% set partition_thread_count_list = [] -%}
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
{% for zope_parameter_dict in zope_partition_dict.values() -%}
{% for zope_parameter_dict in six.itervalues(zope_partition_dict) -%}
{# Apply some zope_parameter_dict default values, to avoid duplication. -#}
{% do zope_parameter_dict.setdefault('thread-amount', 4) -%}
{% do zope_parameter_dict.setdefault('instance-count', 1) -%}
......@@ -53,7 +57,7 @@ config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-m
name = {{ name }}
software-type = {{ software_type }}
return = {{ ' '.join(ret) }}
{% for ret, publish in ret.iteritems() -%}
{% for ret, publish in six.iteritems(ret) -%}
{% if publish -%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
{% endif -%}
......@@ -62,10 +66,10 @@ return = {{ ' '.join(ret) }}
{% endif -%}
{% endfor -%}
{{ root_common.sla(name) }}
{% for k, v in config.iteritems() -%}
{% for k, v in six.iteritems(config) -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
{% for k, v in key_config.iteritems() -%}
{% for k, v in six.iteritems(key_config) -%}
config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
config-name = {{ name }}
......@@ -76,6 +80,7 @@ recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
service-on-watch = ${buildout:directory}/etc/service
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup-caucased = ${:srv}/backup/caucased
{% if not caucase_url -%}
......@@ -180,7 +185,7 @@ connection-url = smtp://127.0.0.2:0/
{% endfor -%}
{% do assert(len(mountpoints) == len(zodb_dict)) -%}
{% set neo = [] -%}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% for server_type, server_dict in six.iteritems(storage_dict) -%}
{% if server_type == 'neo' -%}
{% set ((name, server_dict),) = server_dict.items() -%}
{% do neo.append(server_dict.get('cluster')) -%}
......@@ -199,10 +204,10 @@ connection-url = smtp://127.0.0.2:0/
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% for server_type, server_dict in six.iteritems(storage_dict) -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-name = {{ list(server_dict.keys())[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
......@@ -266,14 +271,14 @@ config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout', 300) -%}
{% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%}
{% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
{% for custom_name, zope_parameter_dict in six.iteritems(zope_partition_dict) -%}
{% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%}
{% set promise_software_url_section_name = 'promise-software-url' ~ partition_name -%}
......@@ -298,12 +303,12 @@ config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 1)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-with-max-rlimit-nofile = {{ dumps(slapparameter_dict.get('with-max-rlimit-nofile', false)) }}
config-with-max-rlimit-nofile = {{ dumps(with_max_rlimit_nofile_enable) }}
{# BBB: zope_parameter_dict used to contain 'webdav', so fallback to it -#}
config-webdav = {{ dumps(current_zope_family_override_dict.get('webdav', zope_parameter_dict.get('webdav', False))) }}
config-xml-rpc = {{ dumps(current_zope_family_override_dict.get('xml-rpc', False)) }}
......@@ -503,7 +508,7 @@ recipe = slapos.cookbook:publish.serialised
Pick any published hosts-dict, they are expected to be identical - and there is
no way to check here.
-#}
hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
hosts-dict = {{ '${' ~ next(iter(zope_address_list_id_dict)) ~ ':connection-hosts-dict}' }}
{% endif -%}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
......
software/slapos-master/software-py3.cfg 0 → 100644
View file @ 0a3fd404
[buildout]
extends = ../wendelin/software-py3.cfg
base.cfg
buildout.hash.cfg
software/slapos-master/software-py3.cfg.json 0 → 100644
View file @ 0a3fd404
{
"name": "SlapOS Master",
"description": "SlapOS Master",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"software-type": "default",
"request": "../erp5/instance-erp5-input-schema.json",
"response": "../erp5/instance-erp5-output-schema.json",
"index": 0
}
}
}
software/slapos-master/software.cfg
View file @ 0a3fd404
......@@ -2,64 +2,4 @@
extends =
../../software/wendelin/software.cfg
buildout.hash.cfg
[erp5-defaults]
wcfs-enable-default = true
[local-bt5-repository]
# Same as bt5-repository, but only local repository.
# Used to generate bt5lists.
list += ${vifib:location}/master/bt5
[default-bt5]
list =
erp5_full_text_mroonga_catalog
slapos_configurator
[erp5_repository_list]
repository_id_list += vifib/master
[erp5]
branch = erp5-vifib
[vifib]
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos-bin.git
branch = master
[eggs]
eggs +=
slapos.core
dummy +=
${vifib:location}
${slapos-bin:location}
extra-paths +=
${vifib:location}/master
${slapos-bin:location}
### Overwrite recipes to introduce customized changes
[download-base-part]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
[template-erp5]
< = download-base-part
filename = instance-erp5.cfg.in
[template-balancer]
< = download-base-part
filename = instance-balancer.cfg.in
[template-apache-backend-conf]
url = ${:_profile_base_location_}/${:filename}
[template-haproxy-cfg]
url = ${:_profile_base_location_}/${:filename}
base.cfg
software/wendelin/base.cfg 0 → 100644
View file @ 0a3fd404
[eggs]
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
extra-paths +=
${wendelin:location}
eggs +=
astor
${scipy:egg}
${msgpack-python:egg}
${msgpack-numpy:egg}
${ipython:egg}
[generic_testrunner_init]
recipe = slapos.recipe.build
init =
import struct
gcc = self.buildout['gcc']
options['initialization'] %= (
gcc['prefix'] + '/lib%s/libstdc++.so' % (8 * struct.calcsize("P"))
) if 'part' in gcc else ''
initialization =
# The 4 lines below will replace the process with another one, with the proper
# LD_PRELOAD environment variable. This is necessary because LD_PRELOAD is only
# taken into account when the process starts. Modifying it in runtime doesn't
# work.
import os
if not os.getenv('LD_PRELOAD'):
os.environ['LD_PRELOAD'] = %r
os.execve(os.path.realpath(__file__), sys.argv, os.environ)
[test_suite_runner]
# we need to override the test suite runner to add our custom libstdc++ dynamic
# library to the path
initialization +=
${generic_testrunner_init:initialization}
[testrunner]
# we need to override the test suite runner to add our custom libstdc++ dynamic
# library to the path
initialization +=
${generic_testrunner_init:initialization}
[erp5_repository_list]
repository_id_list += wendelin
[default-bt5]
list =
erp5_full_text_mroonga_catalog
erp5_wendelin_configurator
[local-bt5-repository]
list += ${wendelin:location}/bt5
# Jupyter is by default enabled in Wendelin
# and also the soft limit of open file descriptors is set
# to the hard limit.
[erp5-defaults]
jupyter-enable-default = true
with-max-rlimit-nofile-enable-default = true
[wendelin]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master
software/wendelin/software-py3.cfg 0 → 100644
View file @ 0a3fd404
[buildout]
versions = versions
extends =
../../component/msgpack-python/buildout.cfg
../../component/msgpack-numpy/buildout.cfg
../../component/scipy/buildout.cfg
../../software/erp5/software-py3.cfg
base.cfg
parts +=
wendelin
scipy
msgpack-python
msgpack-numpy
ipython
jupyter
jupyter-notebook-initialized-scripts
[versions]
msgpack-numpy = 0.4.4.3
software/wendelin/software-py3.cfg.json 0 → 100644
View file @ 0a3fd404
{
"name": "Wendelin",
"description": "Wendelin",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"software-type": "default",
"request": "../erp5/instance-erp5-input-schema.json",
"response": "../erp5/instance-erp5-output-schema.json",
"index": 0
}
}
}
software/wendelin/software.cfg
View file @ 0a3fd404
......@@ -5,6 +5,8 @@ extends =
../../component/msgpack-numpy/buildout.cfg
../../component/scipy/buildout.cfg
../../software/erp5/software.cfg
base.cfg
parts +=
wendelin
scipy
......@@ -14,71 +16,5 @@ parts +=
jupyter
jupyter-notebook-initialized-scripts
[eggs]
initialization =
import scipy.spatial.ckdtree # load our own libstdc++ explicitly at the very beginning
extra-paths +=
${wendelin:location}
eggs +=
astor
${scipy:egg}
${msgpack-python:egg}
${msgpack-numpy:egg}
${ipython:egg}
[generic_testrunner_init]
recipe = slapos.recipe.build
init =
import struct
gcc = self.buildout['gcc']
options['initialization'] %= (
gcc['prefix'] + '/lib%s/libstdc++.so' % (8 * struct.calcsize("P"))
) if 'part' in gcc else ''
initialization =
# The 4 lines below will replace the process with another one, with the proper
# LD_PRELOAD environment variable. This is necessary because LD_PRELOAD is only
# taken into account when the process starts. Modifying it in runtime doesn't
# work.
import os
if not os.getenv('LD_PRELOAD'):
os.environ['LD_PRELOAD'] = %r
os.execve(os.path.realpath(__file__), sys.argv, os.environ)
[test_suite_runner]
# we need to override the test suite runner to add our custom libstdc++ dynamic
# library to the path
initialization +=
${generic_testrunner_init:initialization}
[testrunner]
# we need to override the test suite runner to add our custom libstdc++ dynamic
# library to the path
initialization +=
${generic_testrunner_init:initialization}
[erp5_repository_list]
repository_id_list += wendelin
[default-bt5]
list =
erp5_full_text_mroonga_catalog
erp5_wendelin_configurator
[local-bt5-repository]
list += ${wendelin:location}/bt5
# Jupyter is by default enabled in Wendelin
# and also the soft limit of open file descriptors is set
# to the hard limit.
[erp5-defaults]
jupyter-enable-default = true
with-max-rlimit-nofile-enable-default = true
[wendelin]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master
[versions]
msgpack-numpy = 0.4.4.3
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7