Commit c97b4c31 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents d8b4551d 6e62c95b
......@@ -6,7 +6,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/babeld.git
branch = master
git-executable = ${git:location}/bin/git
revision = hmac-nxd1
revision = hmac-nxd2
[babeld]
recipe = slapos.recipe.cmmi
......
......@@ -8,6 +8,7 @@ parts = dcron-output
[dcron]
recipe = slapos.recipe.cmmi
shared = false
url = http://www.jimpryor.net/linux/releases/dcron-4.4.tar.gz
md5sum = 02d848ba043a9df5bf2102a9f4bc04bd
configure-command = true
......
......@@ -9,6 +9,7 @@ extends =
[mosh]
recipe = slapos.recipe.cmmi
shared = true
url = https://mosh.org/mosh-1.3.0.tar.gz
md5sum = d961276995936953bf2d5a794068b076
configure-options =
......
......@@ -8,8 +8,10 @@ extends =
[nano]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.nano-editor.org/dist/v2.8/nano-2.8.4.tar.xz
md5sum = 02ff28870194178595b287fc16fa611b
location = @@LOCATION@@
# The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
# are installed on the system.
environment=
......@@ -17,7 +19,6 @@ environment=
NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
CPPFLAGS=-I${file:location}/include
LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
location = ${buildout:parts-directory}/${:_buildout_section_name_}
post-install =
cd ${:location} && mkdir etc &&
echo include "${:location}/share/nano/*.nanorc" > etc/nanorc
......@@ -12,11 +12,13 @@ parts = nginx-output
[nginx-common]
recipe = slapos.recipe.cmmi
shared = false
url = https://nginx.org/download/nginx-1.17.1.tar.gz
md5sum = 51021f3e8204a5fc809f5e695a4508db
[nginx]
<= nginx-common
shared = true
configure-options=
--with-http_ssl_module
--with-http_v2_module
......
......@@ -15,9 +15,14 @@ parts =
[nodejs]
<= nodejs-8.9.4
[nodejs-10.6.0]
<= nodejs-base
openssl_location = ${openssl:location}
version = v10.6.0
md5sum = 9df233b86244ebda1ded1f91694fbe86
[nodejs-8.9.4]
<= nodejs-base
recipe = slapos.recipe.cmmi
version = v8.9.4
md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
......@@ -26,24 +31,27 @@ md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
version = v8.6.0
md5sum = 0c95e08220667d8a18b97ecec8218ac6
[nodejs-base]
# Server-side Javascript.
version =
md5sum =
openssl_location = ${openssl-1.0:location}
recipe = slapos.recipe.cmmi
shared = true
url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
configure-options =
--shared-openssl
--shared-openssl-includes=${openssl-1.0:location}/include
--shared-openssl-libpath=${openssl-1.0:location}/lib
--shared-openssl-includes=${:openssl_location}/include
--shared-openssl-libpath=${:openssl_location}/lib
environment =
HOME=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=@@LOCATION@@
PATH=${gcc:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin/:%(PATH)s
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig/
PKG_CONFIG_PATH=${:openssl_location}/lib/pkgconfig/
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${openssl-1.0:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LD_LIBRARY_PATH=${openssl-1.0:location}/lib
LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${:openssl_location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LD_LIBRARY_PATH=${:openssl_location}/lib
[nodejs-8.6.0-output]
# Shared binary location to ease migration
......
......@@ -15,7 +15,9 @@ parts =
[openssh]
recipe = slapos.recipe.cmmi
shared = true
md5sum = 68ba883aff6958297432e5877e9a0fe2
location = @@LOCATION@@
url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
patch-binary = ${patch:location}/bin/patch
patch-options = -p1
......@@ -25,9 +27,9 @@ environment =
CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
--prefix=${:location}
--exec-prefix=${:location}
--with-privsep-path=${:location}/var/empty
[openssh-output]
# Shared binary location to ease migration
......
......@@ -3,10 +3,12 @@ parts = protobuf
[protobuf]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
md5sum = 0820cc2e56d71aef8e99794fcbd184cd
[protobuf-cpp]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
md5sum = 6d59dad503bea5ad420fd09ddad84481
......@@ -7,9 +7,10 @@ parts =
[rsync]
recipe = slapos.recipe.cmmi
shared = true
url = https://download.samba.org/pub/rsync/src/rsync-3.1.3.tar.gz
md5sum = 1581a588fde9d89f6bc6201e8129afaf
make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
PREFIX=@@LOCATION@@
environment =
PATH=${perl:location}/bin:%(PATH)s
......@@ -7,6 +7,7 @@ extends =
[screen]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
version = 4.5.1
md5sum = a8c5da2f42f8a18fa4dada2419d1549b
......
......@@ -9,6 +9,7 @@ extends =
[texinfo4]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/gnu/texinfo/texinfo-4.13.tar.gz
md5sum = 71ba711519209b5fb583fed2b3d86fcb
configure-options =
......
......@@ -7,6 +7,7 @@ extends =
[tig]
recipe = slapos.recipe.cmmi
shared = true
url = http://jonas.nitro.dk/tig/releases/tig-2.1.tar.gz
md5sum = d6c237aba2c03d85897da79789fd6104
environment =
......
......@@ -8,6 +8,7 @@ extends =
[vim]
recipe = slapos.recipe.cmmi
shared = true
url = ftp://ftp.vim.org/pub/vim/unix/vim-8.0.586.tar.bz2
md5sum = b35e794140c196ff59b492b56c1e73db
environment=
......
......@@ -3,8 +3,8 @@ parts = zip
[zip]
recipe = slapos.recipe.cmmi
shared = true
url = http://downloads.sourceforge.net/project/infozip/Zip%203.x%20%28latest%29/3.0/zip30.tar.gz
md5sum = 7b74551e63f8ee6aab6fbc86676c0d37
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = true
make-options = prefix=${:location} NO_BZIP2_SUPPORT=1 -f unix/Makefile generic
make-options = prefix=@@LOCATION@@ NO_BZIP2_SUPPORT=1 -f unix/Makefile generic
......@@ -4,9 +4,7 @@
"additionalProperties": false,
"definitions": {
"tcpv4port": {
"minimum": 0,
"maximum": 65535,
"type": "integer"
"$ref": "./schemas-definitions.json#/tcpv4port"
}
},
"properties": {
......@@ -194,16 +192,34 @@
},
"kumofs": {
"description": "Persistent memcached service",
"additionalProperties": {
"allOf": [
{
"$ref": "./instance-kumofs-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2000
}
}
}
],
"type": "object"
},
"memcached": {
"description": "Volatile memcached service",
"additionalProperties": {
"allOf": [
{
"$ref": "./instance-kumofs-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2010
}
}
}
],
"type": "object"
},
"cloudooo-url": {
......@@ -218,16 +234,34 @@
},
"smtp": {
"description": "Mail queuing and relay service",
"additionalProperties": {
"allOf": [
{
"$ref": "./instance-smtp-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2010
}
}
}
],
"type": "object"
},
"mariadb": {
"description": "Relational database service",
"additionalProperties": {
"allOf": [
{
"$ref": "./instance-mariadb-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2099
}
}
}
],
"type": "object"
},
"zodb-zeo": {
......@@ -257,7 +291,7 @@
"type": "object"
},
"zodb": {
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
"items": {
"required": [
"type"
......@@ -282,8 +316,13 @@
"type": "boolean"
}
},
"patternProperties": {
".!$": {
"$ref": "#/properties/zodb/items/patternProperties/.!$"
}
},
"additionalProperties": {
"type": "string"
"$ref": "#/properties/zodb/items/additionalProperties"
},
"type": "object"
},
......@@ -313,14 +352,39 @@
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
"$ref": "../neoppod/instance-neo-input-schema.json"
"$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
}
}
}
],
"additionalProperties": {
"patternProperties": {
".!$": {
"description": "Override with the value of the first item whose zope id matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"type": [
"integer",
"string"
]
}
],
"type": "array"
},
"type": "array"
}
},
"additionalProperties": {
"type": [
"integer",
"string"
]
},
"type": "object"
},
"type": "array"
......
......@@ -8,7 +8,7 @@
"tcpv4-port": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward"
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"required": [
"tcpv4-port"
],
"type": "object",
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going downward"
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"extends": "./schema-definitions.json#",
"required": [
"tcpv4-port"
],
......@@ -9,7 +8,7 @@
"tcpv4-port": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward"
......
{
"$schema": "http://json-schema.org/draft-07/schema#",
"tcpv4port": {
"minimum": 0,
"maximum": 65535,
"type": "integer"
}
}
......@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""Return the output paramters from the root partition"""
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
......@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = [os.path.basename(path) for path in
glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
bin_programs = map(os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
......@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted([
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
]))
))
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
@classmethod
def getInstanceParameterDict(cls):
# The following example includes the most commonly used options,
# but not necessarily in a meaningful way.
return {'_': json.dumps({
"zodb": [{
"type": "zeo",
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
"storage-dict": {
"cache-size!": [
("a-.*", "50MB"),
],
},
}],
"zope-partition-dict": {
"a": {
"instance-count": 3,
},
"bb": {
"instance-count": 5,
"port-base": 2300,
},
},
})}
def test_zope_conf(self):
zeo_addr = json.loads(
self.getComputerPartition('zodb').getConnectionParameter('_')
)["storage-dict"]["root"]["server"]
def checkParameter(line, kw):
k, v = line.split()
self.assertFalse(k.endswith('!'), k)
try:
expected = kw.pop(k)
except KeyError:
if k == 'server':
return
self.assertIsNotNone(expected)
self.assertEqual(str(expected), v)
def checkConf(zodb, storage):
zodb["mount-point"] = "/"
zodb["pool-size"] = 4
zodb["pool-timeout"] = "10m"
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
i = conf.index("<zodb_db root>") + 1
conf = iter(conf[i:conf.index("</zodb_db>", i)])
for line in conf:
if line == '<zeoclient>':
for line in conf:
if line == '</zeoclient>':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
......@@ -14,7 +14,7 @@
# not need these here).
[deploy-script-controller-script]
filename = deploy-script-controller
md5sum = f0f5dd379361eb37f84e0bc7639f645f
md5sum = 8288e59eb442c662544daffbf446a033
[template-deploy-test]
filename = instance.cfg.in
......
......@@ -72,6 +72,8 @@ function add_checks ()
echo 'iptables-save' >> $LOG_FILE 2>&1
iptables-save >> $LOG_FILE 2>&1
for f in /tmp/playbook-* ; do echo $f ; cat $f; echo; done >> $LOG_FILE 2>&1
echo 'slapos node status' >> $LOG_FILE 2>&1
slapos node status >> $LOG_FILE 2>&1
}
function upload ()
{
......
......@@ -66,7 +66,7 @@ md5sum = a56a44e96f65f5ed20211bb6a54279f4
[nginx-gitlab-http.conf.in]
_update_hash_filename_ = template/nginx-gitlab-http.conf.in
md5sum = abcc5eda03e10b26c74619f299a7f6a8
md5sum = e74695aa1be60f0ffac64ddbe1c8eaf1
[nginx.conf.in]
_update_hash_filename_ = template/nginx.conf.in
......
......@@ -121,6 +121,13 @@ server {
# Set CORS header
add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
add_header 'Access-Control-Allow-Credentials' true;
if ($request_method = OPTIONS ) {
add_header Allow "GET, OPTIONS";
add_header Content-Type text/plain;
add_header 'Access-Control-Allow-Origin' $http_origin;
add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Authorization, Content-Type, Accept";
return 200;
}
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
{{ 'gzip off;' if cfg_https else ''}}
......
......@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd]
filename = instance-nbd.cfg.in
md5sum = a05b581d65768ac55faf3b06d4aec447
md5sum = c3667c9bbaa8627dad90f1abd62d2f16
[template-ansible-promise]
filename = template/ansible-promise.in
......
......@@ -384,8 +384,7 @@
"title": "NBD hostname or IP",
"description": "hostname (or IP) of the NBD server containing the boot image.",
"type": "string",
"format": "internet-address",
"default": "debian.nbd.vifib.net"
"format": "internet-address"
},
"nbd-port": {
"title": "NBD port",
......
......@@ -64,8 +64,7 @@
"title": "NBD hostname or IP",
"description": "hostname (or IP) of the NBD server containing the boot image.",
"type": "string",
"format": "internet-address",
"default": "debian.nbd.vifib.net"
"format": "internet-address"
},
"nbd-port": {
"title": "NBD port",
......
......@@ -172,8 +172,7 @@
"title": "NBD hostname",
"description": "hostname (or IP) of the NBD server containing the boot image.",
"type": "string",
"format": "internet-address",
"default": "debian.nbd.vifib.net"
"format": "internet-address"
},
"nbd-port": {
"title": "NBD port",
......
......@@ -8,7 +8,7 @@ parts =
nbd-promise
onetimeupload-promise
publish-connection-information
extends = ${monitor-template:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
......
......@@ -14,11 +14,11 @@
# not need these here).
[instance-common]
filename = instance-common.cfg.in
md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
md5sum = 80599fcc6e5d07270d7900aebfd62139
[root-common]
filename = root-common.cfg.in
md5sum = ccc6e33412259415ec6c3452d37b77cc
md5sum = c03fbfc9df9edc1ef60be970e0627c5e
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
......@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
[template-neo]
filename = instance.cfg.in
md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
md5sum = 9e63e16eda75e73ad4ffb50afde0505d
[cluster]
filename = cluster.cfg.in
......
......@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extensions = jinja2.ext.do
extensions =
jinja2.ext.do
jinja2.ext.loopcontrols
extra-context =
context =
key ipv4_set slap-configuration:ipv4
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
"definitions": {
"neo-cluster": {
"additionalProperties": false,
"require": [
"cluster"
],
"properties": {
"cluster": {
"description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
......@@ -72,10 +71,13 @@
"_key": {
"type": "string"
},
"node-list": {
"description": "List of dictionaries containing parameters for each node.",
"items": {
"description": "Dictionary containing parameters required to configure individual nodes.",
"node-count": {
"description": "Number of nodes to deploy.",
"default": 1,
"type": "integer"
},
"node": {
"description": "Default node parameters.",
"default": {},
"additionalProperties": false,
"properties": {
......@@ -135,8 +137,41 @@
},
"type": "object"
},
"node!": {
"description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string"
},
{
"allOf": [
{
"$ref": "#/definitions/neo-cluster/properties/node"
},
{
"description": "Override value (parameters for maching nodes)."
}
]
}
],
"type": "array"
},
"type": "array"
}
},
"type": "object"
}
},
"allOf": [
{
"$ref": "#/definitions/neo-cluster"
},
{
"required": [
"cluster"
]
}
]
}
......@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
<= jinja2-template-base
template = {{ cluster }}
extra-context =
import re re
import urlparse urlparse
import-list =
rawfile root_common {{ root_common }}
......
......@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
{% macro apply_overrides(dict_, reference) -%}
{% for key in list(dict_) -%}
{% if key.endswith('!') -%}
{% for pattern, value in dict_.pop(key, ()) -%}
{% set m = re.match(pattern, reference) -%}
{% if m and m.group() == reference %}{# PY3: fullmatch -#}
{% do dict_.__setitem__(key[:-1], value) -%}
{% break -%}
{% endif -%}
{% endfor -%}
{% endif -%}
{% endfor -%}
{% endmacro -%}
{% macro common_section() -%}
[request-common-base]
recipe = slapos.cookbook:request.serialised
......@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
software-type = {{ software_type }}
{% set node_list = parameter_dict.get('node-list', ({},)) -%}
{% set node_list = parameter_dict.get('node-list') -%}
{% if node_list == None -%}
{% set node_list = [] -%}
{% for i in range(parameter_dict.get('node-count', 1)) -%}
{% set x = parameter_dict.copy() -%}
{% do apply_overrides(x, prefix ~ i) -%}
{% do node_list.append(x.get('node', {})) -%}
{% endfor -%}
{% else %}{# BBB -#}
{% do assert('node-count' not in parameter_dict) -%}
{% do assert('node' not in parameter_dict) -%}
{% do assert('node!' not in parameter_dict) -%}
{% endif -%}
{% do assert(node_list) -%}
{% set storage_count = [] -%}
{% for node in node_list -%}
{% do storage_count.append(node.get('storage-count', 1)) -%}
......
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 3d12778d8dc910180ff3a7fcae8b37f9
md5sum = 4664f7dae66d3f582e34cec2ca627501
......@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
working-dir = $${buildout:directory}/tmp
[test-list]
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
......
......@@ -102,6 +102,11 @@ setup = ${slapos-repository:location}/software/nextcloud/test/
egg = slapos.test.nextcloud
setup = ${slapos-repository:location}/software/turnserver/test/
[slapos.test.theia-setup]
<= setup-develop-egg
egg = slapos.test.theia
setup = ${slapos-repository:location}/software/theia/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......
# Theia software release
Theia is a cloud (and desktop) IDE https://www.theia-ide.org
This version comes pre-configured with a few plugins, but does not come with python plugin, to let
you choose between theia and vscode one.
## Installing vscode extensions
To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
Check theia's log files in case of issues.
see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
## jedi
[jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has
some support for `zc.buildout`. It looks up for a `buildout.cfg` file and if found will load all scripts
from the bin directory from this buildout to add eggs to sys.path. In webrunner we have almost 100 scripts
in bin directory, with maybe 30 eggs in each scripts, so this makes jedi so slow it's unusable. Also, if
an error occurs parsing these scripts, jedi won't be usable. This issue is tracked in
https://github.com/davidhalter/jedi/issues/1325
A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
\ No newline at end of file
......@@ -15,12 +15,12 @@
[instance]
filename = instance.cfg.in
md5sum = 6a0a65ff9f81f907d53c42c65b54442a
md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
[yarn.lock]
filename = yarn.lock
md5sum = fc2f20d496911241a0d9001c445d6158
md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
[python-language-server-requirements.txt]
filename = python-language-server-requirements.txt
md5sum = 242c8359be6893725ad2ac8e3c864eb0
md5sum = ce87c9818d64a16fce49c84ed5dba8fc
......@@ -2,7 +2,6 @@
parts =
monitor-base
promises
.bashrc
frontend-reload
publish-connection-parameter
......@@ -79,37 +78,15 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
env -i HOME=$${directory:home} LC_ALL=C.UTF-8 USER=$${user:pw-name} LOGNAME=$${user:pw-name} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
ip = $${instance-parameter:ipv4-random}
hostname = $${:ip}
port = 3000
base-url = http://$${:hostname}:$${:port}/
[.bashrc]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:home}/$${:_buildout_section_name_}
template = inline:
export PS1="$ " # because we are in a gowork workspace
# XXX .bash_profile is not executed, so we introduce a bashrc.theia file
# to allow customizations.
if [ -f "$HOME/.bashrc.theia" ] ; then
source $HOME/.bashrc.theia
fi
# If not running interactively, don't do anything
case $- in
*i*) ;;
*) return;;
esac
# enable color support
if [ -x /usr/bin/dircolors ]; then
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
alias ls='ls --color=auto'
fi
# common ls aliases
alias ll='ls -l'
alias la='ls -A'
alias l='ls -CF'
hash-existing-files =
${yarn.lock:output}
${theia-wrapper:rendered}
[promises]
recipe =
......
......@@ -20,4 +20,4 @@ snowballstemmer==1.2.1
toml==0.10.0
typed-ast==1.2.0
typeshed==0.0.1
yapf==0.25.0
yapf==0.28.0
......@@ -24,7 +24,7 @@ parts =
[nodejs]
<= nodejs-8.9.4
<= nodejs-10.6.0
[python3]
<= python3.6.6
......@@ -34,10 +34,10 @@ parts =
# this could become a component, but it needs to be invoked from nodejs explicitly,
# otherwise it uses system's nodejs
# XXX why don't we build a wrapper ?
version = 1.11.0
version = 1.17.3
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
md5sum = d4f05075f534dd9a0a8c18c650b55f0d
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[python-language-server]
......@@ -60,8 +60,9 @@ mode = 0644
[theia]
recipe = plone.recipe.command
command = ${bash:location}/bin/bash -c "
export PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
mkdir -p ${:location} && \
mkdir -p \$TMPDIR && \
cd ${:location} && \
cp ${package.json:rendered} . &&
cp ${yarn.lock:output} . &&
......@@ -69,6 +70,7 @@ command = ${bash:location}/bin/bash -c "
${yarn:location}/bin/yarn theia build"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
uses = ${yarn.lock:recipe}
[yarn.lock]
recipe = slapos.recipe.template
......@@ -78,25 +80,54 @@ mode = 0644
[package.json]
recipe = slapos.recipe.template:jinja2
# this comes from https://github.com/theia-ide/theia-apps/blob/598d4dc9d4f9e0514869273c069f734a41f20207/theia-full-docker/next.package.json
# this comes from https://github.com/theia-ide/theia-apps/blob/def6d29a5fb3739aebc17c6a5b67f1b5c16caacd/theia-full-docker/next.package.json
# but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
template =
inline:
{
"private": true,
"theia": {
"frontend": {
"config": {
"applicationName": "Theia SlapOS",
"preferences": {
"files.enableTrash": false,
"files.exclude": {
"**.pyc": true,
"**.egg-info": true,
"__pycache__": true,
".git": true,
".env": true
},
"files.watcherExclude": {
"**/.eggs/**": true,
"**/.env/**": true,
"**/.git/**": true,
"**/node_modules/**": true
},
"editor.multiCursorModifier": "ctrlCmd",
"plantuml.webservice": "//plantuml.host.vifib.net/svg/",
"plantuml.monochrome": false,
"editor.tabSize": 2
}
}
}
},
"dependencies": {
"typescript": "latest",
"@theia/callhierarchy": "next",
"@theia/core": "next",
"@theia/cpp": "next",
"@theia/debug": "next",
"@theia/debug-nodejs": "next",
"@theia/docker": "next",
"@theia/editor": "next",
"@theia/editorconfig": "next",
"@theia/extension-manager": "next",
"@theia/editor-preview": "next",
"@theia/file-search": "next",
"@theia/filesystem": "next",
"@theia/git": "next",
"@theia/go": "next",
"@theia/getting-started": "next",
"@theia/java": "next",
"@theia/json": "next",
"@theia/keymaps": "next",
......@@ -111,10 +142,12 @@ template =
"@theia/outline-view": "next",
"@theia/output": "next",
"@theia/plantuml": "next",
"@theia/plugin": "next",
"@theia/plugin-ext": "next",
"@theia/plugin-ext-vscode": "next",
"@theia/preferences": "next",
"@theia/preview": "next",
"@theia/process": "next",
"@theia/python": "next",
"@theia/ruby": "next",
"@theia/rust": "next",
"@theia/search-in-workspace": "next",
......@@ -126,7 +159,8 @@ template =
"@theia/userstorage": "next",
"@theia/variable-resolver": "next",
"@theia/workspace": "next",
"theia-yang-extension": "next"
"theia-yang-extension": "next",
"typescript": "latest"
},
"devDependencies": {
"@theia/cli": "next"
......@@ -169,6 +203,8 @@ template =
export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
. ${gowork:env.sh}
export SHELL=bash
# reset PS1 from gowork
export PS1='$ '
cd ${theia:location}
exec ${yarn:location}/bin/yarn theia start $@
......
Tests for Theia software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.theia'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Theia",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import textwrap
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestTheia(SlapOSInstanceTestCase):
def setUp(self):
self.theia_url = self.computer_partition.getConnectionParameterDict(
)['url']
def test_http_get(self):
resp = requests.get(self.theia_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
# without login/password, this is unauthorized
parsed_url = urlparse(self.theia_url)
resp = requests.get(
parsed_url._replace(
netloc='[{}]:{}'.format(
parsed_url.hostname,
parsed_url.port)).geturl(),
verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
This diff is collapsed.
......@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
[template-zope-conf]
filename = zope.conf.in
md5sum = 114e0ac43281b943931754ed317ebc36
md5sum = 762897486b1e7e28b614224a9a577125
[site-zcml]
filename = site.zcml
......@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template]
filename = instance.cfg.in
md5sum = f6c6820f9b3653d0d5c29708606fc591
md5sum = 520b6bf3461dddc9c8b862e50b14465d
[monitor-template-dummy]
filename = dummy.cfg
......@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zope]
filename = instance-zope.cfg.in
md5sum = b1685783f4c93da918ccc83702559e6f
md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
[zope-conf-base]
< = jinja2-template-base
template = {{ parameter_dict['zope-conf-template'] }}
extensions =
jinja2.ext.do
jinja2.ext.loopcontrols
import-list =
rawfile root_common {{ root_common }}
{% macro zope(
index,
......@@ -312,10 +317,10 @@ longrequest-logger-file =
[{{ conf_name }}]
< = zope-conf-base
rendered = ${directory:etc}/{{ name }}.conf
extensions = jinja2.ext.do
context =
section parameter_dict {{ conf_parameter_name }}
import os os
import re re
[{{ section(name) }}]
< = runzope-base
......
......@@ -46,6 +46,7 @@ extra-context =
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
import urlparse urlparse
import-list =
file root_common context:root-common
......@@ -111,6 +112,7 @@ template = {{ template_zope }}
filename = instance-zope.cfg
extra-context =
key buildout_directory buildout:directory
key root_common context:root-common
section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse
import hashlib hashlib
......
{% set slapparameter_dict = {} %}{# dummy -#}
{% import "root_common" as root_common with context -%}
{% set node_id = parameter_dict['node-id'] -%}
# Note: Environment is setup in running wrapper script, as zope.conf is read
# too late for some components.
%define INSTANCE {{ parameter_dict['instance'] }}
......@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
{% endif -%}
<product-config CMFActivity>
node-id {{ parameter_dict['node-id'] }}
node-id {{ node_id }}
</product-config>
{% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
......@@ -145,10 +148,12 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
<zodb_db {{ name }}>
{%- set storage_type = type_dict[zodb_dict.pop('type')] %}
{%- set storage_dict = zodb_dict.pop('storage-dict') %}
{%- do root_common.apply_overrides(zodb_dict, node_id) %}
{%- for key, value in zodb_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
<{{ storage_type }}>
{%- do root_common.apply_overrides(storage_dict, node_id) %}
{%- for key, value in storage_dict.iteritems() %}
{{ key }} {{ value }}
{%- endfor %}
......
......@@ -140,11 +140,11 @@ slapos.cookbook = 1.0.123
slapos.core = 1.5.4
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.19
slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.2
slapos.recipe.build = 0.42
slapos.recipe.cmmi = 0.11
slapos.toolbox = 0.98
slapos.recipe.cmmi = 0.12
slapos.toolbox = 0.100
stevedore = 1.21.0
subprocess32 = 3.5.3
unicodecsv = 0.14.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment