Commit 4475ff4b authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 910d3827 722ecd22
......@@ -12,11 +12,11 @@ parts =
[ca-certificates]
recipe = slapos.recipe.cmmi
shared = true
url = http://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20190110.tar.xz
md5sum = e91d3d9259127ba2dbb65fda58d73f31
url = http://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20210119.tar.xz
md5sum = c02582bf9ae338e558617291897615eb
patch-binary = ${patch:location}/bin/patch
patches =
${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
${:_profile_base_location_}/ca-certificates-any-python.patch#47c2509f1346bd5af8123fb1a2751c2c
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
patch-options = -p0
configure-command = true
......
......@@ -15,7 +15,7 @@
#
all:
- python certdata2pem.py
- python3 certdata2pem.py
+ for x in '' 2 3; do type python$$x && break; done >/dev/null \
+ && python$$x certdata2pem.py
......
......@@ -34,7 +34,7 @@ depends =
${patch:recipe}
recipe = slapos.recipe.build
# Latest version provided by SlapOS.
part = gcc-8.4
part = gcc-8.5
# Minimum version for all components that might be required for
# slapos.rebootstrap (see https://bugs.python.org/issue34112 about Python 3.7+).
min_version = 5.4
......
......@@ -10,7 +10,7 @@ extends =
../binutils/buildout.cfg
parts =
gcc-8.4
gcc-8.5
[gcc-common]
recipe = slapos.recipe.cmmi
......@@ -75,10 +75,10 @@ patches =
${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
${:_profile_base_location_}/glibc2.30-ipc_perm.patch#563b7f5a38c2ea6bf741c328422b4c98
[gcc-8.4]
[gcc-8.5]
<= gcc-common
version = 8.4.0
md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
version = 8.5.0
md5sum = 0c1f625768840187ef3b10adebe8e3b0
[gcc-10.2]
<= gcc-common
......
[buildout]
extends =
../patch/buildout.cfg
parts =
gdbm
[gdbm]
recipe = slapos.recipe.cmmi
shared = true
version = 1.18
url = ftp://ftp.gnu.org/gnu/gdbm/gdbm-${:version}.tar.gz
md5sum = e316f8e4a3e7e4f23955be65d54fec48
patch-options = -p1
patches =
${:_profile_base_location_}/extern_parseopt.patch#30baeaaa0c6a6e6be8cf56a875726619
version = 1.19
url = http://ftp.gnu.org/gnu/gdbm/gdbm-${:version}.tar.gz
md5sum = aeb29c6a90350a4c959cd1df38cd0a7e
configure-options =
--disable-static
--enable-libgdbm-compat
environment =
PATH=${patch:location}/bin:%(PATH)s
# install as parts/gdbm/include/gdbm/*.h etc. because some softwares
# (eg. python's dbmmodule.c extension) assume the location like this.
includedir = @@LOCATION@@/include
make-targets =
install includedir=${:includedir}/gdbm && rm -f ${:includedir}/*.h && ln -sf gdbm/gdbm.h ${:includedir}/gdbm.h
--- a/src/parseopt.c
+++ b/src/parseopt.c
@@ -255,8 +255,8 @@
}
char *parseopt_program_name;
-char *parseopt_program_doc;
-char *parseopt_program_args;
+extern char *parseopt_program_doc;
+extern char *parseopt_program_args;
const char *program_bug_address = "<" PACKAGE_BUGREPORT ">";
void (*parseopt_help_hook) (FILE *stream);
......@@ -11,8 +11,8 @@ extends =
[gettext]
recipe = slapos.recipe.cmmi
shared = true
url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.8.1.tar.lz
md5sum = d838d2c4144261d0c5fbab4a0aceb5c1
url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.21.tar.lz
md5sum = 65c9a4d0a2d180e3c2afc2b3440382ef
configure-options =
--disable-static
......
......@@ -15,8 +15,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/groonga/groonga-11.0.0.tar.gz
md5sum = 2ea277c19f21b9d45410547173ff41c2
url = https://packages.groonga.org/source/groonga/groonga-11.0.2.tar.gz
md5sum = 753ba6fad77598baf93615c4b9c535b1
groonga-plugin-dir = @@LOCATION@@/lib/groonga/plugins/
# temporary patch to respect more tokens in natural language mode.
patches =
......
......@@ -16,6 +16,7 @@ parts +=
max_version = 0
[jupyter]
extra-eggs =
python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
......@@ -66,7 +67,6 @@ context =
Pygments = 2.2.0
astor = 0.5
backports-abc = 0.5
backports.functools-lru-cache = 1.6.1
backports.shutil-get-terminal-size = 1.0.0
cycler = 0.10.0
ipykernel = 4.5.2
......@@ -122,11 +122,6 @@ pexpect = 4.2.1
# ipython==5.3.0
pickleshare = 0.7.4
# Required by:
# matplotlib==2.1.2
# pandas==0.19.2
python-dateutil = 2.6.0
# Required by:
# pathlib2==2.2.1
scandir = 1.5
......
......@@ -3,6 +3,7 @@ extends =
../numpy/openblas.cfg
../matplotlib/buildout.cfg
../ipython/buildout.cfg
../python-cffi/buildout.cfg
../python-pyzmq/buildout.cfg
../scipy/buildout.cfg
../scikit-learn/buildout.cfg
......@@ -14,6 +15,11 @@ parts =
jupyter
jupyter-notebook-scripts
[argon2-cffi]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = ${python-cffi:egg}
[jupyter-env]
<= numpy-env
......@@ -46,6 +52,9 @@ eggs =
# additional eggs for jupyterlab
jupyterlab
jupyterlab-launcher
${:extra-eggs}
extra-eggs =
${argon2-cffi:egg}
interpreter = pythonjupyter
......
......@@ -7,8 +7,8 @@ parts =
[libexpat]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/libexpat/libexpat/releases/download/R_2_2_9/expat-2.2.9.tar.lz
md5sum = c356e4f2092df4f0b0ffef904f001842
url = https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.lz
md5sum = 074cab91a2f27dc8cd6c282effaa8ffd
configure-options =
--disable-static
--without-xmlwf
......
......@@ -8,13 +8,13 @@ parts =
[libffi]
recipe = slapos.recipe.cmmi
shared = true
version = 3.2.1
url = ftp://sourceware.org/pub/libffi/libffi-${:version}.tar.gz
md5sum = 83b89587607e3eb65c70d361f13bab43
version = 3.3
url = http://sourceware.org/pub/libffi/libffi-${:version}.tar.gz
md5sum = 6313289e32f1d38a9df4770b014a2ca7
location = @@LOCATION@@
patch-options = -p1
patches =
${:_profile_base_location_}/includedir.diff#e430307d16a0d215a24ec2acff23d184
${:_profile_base_location_}/libdir.patch#f7274250c8ca43b10e7fc968f730b833
configure-options =
--disable-static
--enable-portable-binary
......
Index: b/libffi.pc.in
===================================================================
--- a/libffi.pc.in
+++ b/libffi.pc.in
@@ -2,10 +2,10 @@ prefix=@prefix@
exec_prefix=@exec_prefix@
libdir=@libdir@
toolexeclibdir=@toolexeclibdir@
-includedir=${libdir}/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
+includedir=@includedir@
Name: @PACKAGE_NAME@
Description: Library supporting Foreign Function Interfaces
Version: @PACKAGE_VERSION@
-Libs: -L${toolexeclibdir} -lffi
+Libs: -L${libdir} -lffi
Cflags: -I${includedir}
Index: b/include/Makefile.am
===================================================================
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -5,5 +5,5 @@ AUTOMAKE_OPTIONS=foreign
DISTCLEANFILES=ffitarget.h
EXTRA_DIST=ffi.h.in ffi_common.h
-includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
+includesdir = $(includedir)
nodist_includes_HEADERS = ffi.h ffitarget.h
Index: b/include/Makefile.in
===================================================================
--- a/include/Makefile.in
+++ b/include/Makefile.in
@@ -314,7 +314,7 @@ top_srcdir = @top_srcdir@
AUTOMAKE_OPTIONS = foreign
DISTCLEANFILES = ffitarget.h
EXTRA_DIST = ffi.h.in ffi_common.h
-includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include
+includesdir = $(includedir)
nodist_includes_HEADERS = ffi.h ffitarget.h
all: all-am
--- a/libffi.pc.in
+++ b/libffi.pc.in
@@ -7,5 +7,5 @@ prefix=@prefix@
Name: @PACKAGE_NAME@
Description: Library supporting Foreign Function Interfaces
Version: @PACKAGE_VERSION@
-Libs: -L${toolexeclibdir} -lffi
+Libs: -L${libdir} -lffi
Cflags: -I${includedir}
......@@ -12,8 +12,8 @@ parts =
[libxml2]
recipe = slapos.recipe.cmmi
shared = true
url = http://xmlsoft.org/sources/libxml2-2.9.9.tar.gz
md5sum = c04a5a0a042eaa157e8e8c9eabe76bd6
url = http://xmlsoft.org/sources/libxml2-2.9.10.tar.gz
md5sum = 10942a1dc23137a8aa07f0639cbfece5
configure-options =
--disable-static
--without-python
......
......@@ -5,7 +5,7 @@ parts =
[lunzip]
recipe = slapos.recipe.cmmi
shared = true
url = http://download.savannah.gnu.org/releases-redirect/lzip/lunzip/lunzip-1.7.tar.gz
md5sum = cb37bd46590a78bd19a77748b3a097ee
url = http://download.savannah.gnu.org/releases-redirect/lzip/lunzip/lunzip-1.12.tar.gz
md5sum = 09caf2475c58aa40e94b599a4f7a2d13
configure-options =
--disable-static
......@@ -30,8 +30,8 @@ parts =
recipe = slapos.recipe.cmmi
shared = true
url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
version = 10.4.18
md5sum = 22e429d532fcfcb0e4beba64e1c296f0
version = 10.4.19
md5sum = bf60c7a3feac5854745cd1ad5133f09a
location = @@LOCATION@@
pre-configure =
set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
......@@ -96,8 +96,8 @@ post-install =
# as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/mroonga/mroonga-11.00.tar.gz
md5sum = 0aefcbb3c0dc6e29c53c6ee6287114b8
url = https://packages.groonga.org/source/mroonga/mroonga-11.02.tar.gz
md5sum = 0729c74efc92bfc404b88597488d07e9
pre-configure =
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......@@ -131,8 +131,8 @@ environment =
### (we just override here for easier revert)
[mariadb-10.3]
<= mariadb-10.4
version = 10.3.27
md5sum = 6ab2934a671191d8ca8730e9a626c5c9
version = 10.3.29
md5sum = a5adad1c4fb1717d7fe6d608fd4d40de
post-install =
ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
for x in ${lz4:location} ${snappy:location} ${zstd:location}
......
[buildout]
[python-cachecontrol]
recipe = zc.recipe.egg:custom
egg = cachecontrol
setup-eggs =
pbr
lockfile
[buildout]
[python-cliff]
recipe = zc.recipe.egg:custom
egg = cliff
setup-eggs =
pbr
......@@ -77,8 +77,6 @@ eggs =
${lxml-python:egg}
${python-cffi:egg}
${python-PyYAML:egg}
${python-cachecontrol:egg}
${python-cliff:egg}
${python-cryptography:egg}
pyOpenSSL
slapos.libnetworkcache
......
......@@ -8,8 +8,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
shared = true
url = https://sqlite.org/2020/sqlite-autoconf-3320300.tar.gz
md5sum = 2e3911a3c15e85c2f2d040154bbe5ce3
url = https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz
md5sum = d1d1aba394c8e0443077dc9f1a681bb8
configure-options =
--disable-static
--enable-readline
......
......@@ -32,7 +32,7 @@ recipe = zc.recipe.egg
interpreter = python-${:_buildout_section_name_}
eggs =
PyRSS2Gen
${python-dateutil:egg}
python-dateutil
##########################################################
# Service startup scripts and configuration files
......
......@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = a6a626fd1579fd1d4b80ea67433ca16a
md5sum = dd6542a91746e6e3720eaa1590792dbe
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
......@@ -30,7 +30,7 @@ md5sum = 1248911409cbeea980a838b04ee451d2
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 8ce1d5bf09662d941f940be7e6493918
md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......
......@@ -508,7 +508,7 @@ filename = logging.yaml
{%- raw %}
template = inline:
map /HTTPS/ http://{{ ipv4 }}:{{ https_port }}
map / http://{{ ipv4 }}:{{ http_port }}
map /HTTP/ http://{{ ipv4 }}:{{ http_port }}
{%- endraw %}
extra-context =
raw ipv4 {{ instance_parameter_dict['ipv4-random'] }}
......
......@@ -5,7 +5,7 @@
{%- set frontend_slave_list = [] %}
{%- set part_list = [] %}
{%- set cache_port = caddy_configuration.get('cache-port') %}
{%- set cache_access = "http://%s:%s" % (instance_parameter_dict['ipv4-random'], cache_port) %}
{%- set cache_access = "http://%s:%s/HTTP" % (instance_parameter_dict['ipv4-random'], cache_port) %}
{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (instance_parameter_dict['ipv4-random'], cache_port) %}
{%- set backend_haproxy_http_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['http-port']) %}
{%- set backend_haproxy_https_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['https-port']) %}
......
......@@ -1485,6 +1485,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'enable_cache': True,
'disable-via-header': True,
},
'enable_cache-https-only': {
'url': cls.backend_url,
'https-only': False,
'enable_cache': True,
},
'enable-http2-false': {
'url': cls.backend_url,
'enable-http2': False,
......@@ -1681,9 +1686,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '50',
'accepted-slave-amount': '51',
'rejected-slave-amount': '0',
'slave-amount': '50',
'slave-amount': '51',
'rejected-slave-dict': {
},
'warning-slave-dict': {
......@@ -3669,6 +3674,61 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
result.headers['Location']
)
def test_enable_cache_https_only(self):
parameter_dict = self.assertSlaveBase('enable_cache-https-only')
result = fakeHTTPSResult(
parameter_dict['domain'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600'
},
headers
)
result = fakeHTTPResult(
parameter_dict['domain'],
'HTTPS/test', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqual(httplib.OK, result.status_code)
self.assertEqualResultJson(result, 'Path', '/HTTPS/test')
headers = result.headers.copy()
result = fakeHTTPSResult(
parameter_dict['domain'],
'HTTP/test', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqual(httplib.OK, result.status_code)
self.assertEqualResultJson(result, 'Path', '/HTTP/test')
headers = result.headers.copy()
def test_enable_cache(self):
parameter_dict = self.assertSlaveBase('enable_cache')
......
......@@ -55,6 +55,9 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
T-2/var/log/httpd/_enable_cache-https-only_access_log
T-2/var/log/httpd/_enable_cache-https-only_backend_log
T-2/var/log/httpd/_enable_cache-https-only_error_log
T-2/var/log/httpd/_enable_cache_access_log
T-2/var/log/httpd/_enable_cache_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_access_log
......
......@@ -55,6 +55,9 @@ T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
T-2/var/log/httpd/_enable_cache-https-only_access_log
T-2/var/log/httpd/_enable_cache-https-only_backend_log
T-2/var/log/httpd/_enable_cache-https-only_error_log
T-2/var/log/httpd/_enable_cache_access_log
T-2/var/log/httpd/_enable_cache_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_access_log
......
......@@ -46,7 +46,6 @@ dnspython = 1.15.0
PyXML = 0.8.5
WebOb = 1.8.5
WebTest = 2.0.33
backports.functools-lru-cache = 1.5
soupsieve = 1.8
#gitdb2 = 2.0.4
#smmap2 = 2.0.4
......
......@@ -54,13 +54,12 @@ Pygments = 2.7.2
astor = 0.5
async-generator = 1.10
backports-abc = 0.5
backports.functools-lru-cache = 1.6.1
backports.shutil-get-terminal-size = 1.0.0
bleach = 3.2.1
cycler = 0.10.0
defusedxml = 0.6.0
entrypoints = 0.3
ipykernel = 5.3.4
ipykernel = 5.3.4:whl
ipython = 5.3.0
ipython-genutils = 0.1.0
ipywidgets = 6.0.0
......@@ -130,10 +129,6 @@ pickleshare = 0.7.4
# notebook==6.1.5
prometheus-client = 0.9.0
# Required by:
# statsmodels==0.11.1
python-dateutil = 2.8.1
# Required by:
# pathlib2==2.2.1
scandir = 1.5
......
......@@ -32,7 +32,7 @@ import os
import requests
import sqlite3
from slapos.proxy.db_version import DB_VERSION
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -169,27 +169,21 @@ class SelectMixin(object):
)
return sqlite3.connect(sqlitedb_file)
def select(self, fields, table, where={}):
connection = self.sqlite3_connect()
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
connection.row_factory = dict_factory
cursor = connection.cursor()
condition = " AND ".join("%s='%s'" % (k, v) for k, v in where.items())
cursor.execute(
"SELECT %s FROM %s%s"
% (
def select(self, table, fields=('*',), **where):
db = self.sqlite3_connect()
try:
db.row_factory = lambda cursor, row: {
col[0]: row[idx]
for idx, col in enumerate(cursor.description)
}
return db.execute("SELECT %s FROM %s%s%s" % (
", ".join(fields),
table,
" WHERE %s" % condition if where else "",
)
)
return cursor.fetchall()
table, DB_VERSION,
" WHERE " + " AND ".join("%s='%s'" % x for x in where.items())
if where else "",
)).fetchall()
finally:
db.close()
class TestJupyterCustomFrontend(SelectMixin, InstanceTestCase):
......@@ -226,7 +220,7 @@ class TestJupyterCustomFrontend(SelectMixin, InstanceTestCase):
except Exception:
pass
selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
selection = self.select("slave", hosted_by=r._partition_id)
self.assertEqual(len(selection), 1)
......@@ -268,7 +262,7 @@ class TestJupyterCustomAdditional(SelectMixin, InstanceTestCase):
except Exception:
pass
selection = self.select(fields=["*"], table = "slave14", where = {"hosted_by": r._partition_id})
selection = self.select("slave", hosted_by=r._partition_id)
self.assertEqual(len(selection), 1)
......
......@@ -46,6 +46,7 @@ import time
import shutil
import sys
from slapos.proxy.db_version import DB_VERSION
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError
......@@ -214,22 +215,19 @@ class MonitorAccessMixin(object):
return sqlite3.connect(sqlitedb_file)
def get_all_instantiated_partition_list(self):
connection = self.sqlite3_connect()
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
connection.row_factory = dict_factory
cursor = connection.cursor()
cursor.execute(
"SELECT reference, xml, connection_xml, partition_reference, "
"software_release, requested_state, software_type "
"FROM partition14 "
"WHERE slap_state='busy'")
return cursor.fetchall()
db = self.sqlite3_connect()
try:
db.row_factory = lambda cursor, row: {
col[0]: row[idx]
for idx, col in enumerate(cursor.description)
}
return db.execute(
"SELECT reference, xml, connection_xml, partition_reference,"
" software_release, requested_state, software_type"
" FROM partition%s"
" WHERE slap_state='busy'" % DB_VERSION).fetchall()
finally:
db.close()
def test_access_monitor(self):
connection_parameter_dict = self.computer_partition\
......
......@@ -45,7 +45,6 @@ environment = neoppod-setup-env
[neoppod]
recipe = zc.recipe.egg
eggs = neoppod[admin, ctl, master]
${python-dateutil:egg}
${cython-zstd:egg}
${msgpack-python:egg}
${:adapter-egg}
......@@ -133,7 +132,6 @@ pycrypto = 2.6.1
pycurl = 7.43.0
setproctitle = 1.1.10
cython-zstd = 0.2
python-dateutil = 2.7.3
# Required by:
# mock = 3.0.5
......
......@@ -56,6 +56,5 @@ cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
passlib = 1.7.1
GitPython = 2.1.11
lockfile = 0.12.2
feedparser = 5.2.1
atomize = 0.2.0
......@@ -2,7 +2,6 @@
extends =
../../component/bcrypt/buildout.cfg
../../component/python-cliff/buildout.cfg
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
......@@ -202,7 +201,6 @@ setup = ${slapos.core-repository:location}
[python-interpreter]
eggs += ${:extra-eggs}
extra-eggs =
${python-cliff:egg}
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
......@@ -337,7 +335,7 @@ forcediphttpsadapter = 1.0.1
httplib2 = 0.11.3
image = 1.5.25
paramiko = 2.4.2
plantuml = 0.3.0
plantuml = 0.3.0:whl
pysftp = 0.2.9
requests-toolbelt = 0.8.0
selenium = 3.141.0
......@@ -348,7 +346,7 @@ PyPDF2 = 1.26.0+SlapOSPatched001
# Django 1.11 is python 2 compatible
Django = 1.11
mock = 2.0.0
mock = 2.0.0:whl
testfixtures = 6.11
funcsigs = 1.0.2
mysqlclient = 1.3.12
......
......@@ -234,7 +234,7 @@ dnspython = 1.15.0
funcsigs = 1.0.2
httmock = 1.2.6
manuel = 1.9.0
mock = 2.0.0
mock = 2.0.0:whl
testfixtures = 6.11.0
pycurl = 7.43.0.2
pyflakes = 2.0.0
......
......@@ -177,4 +177,3 @@ prettytable = 0.7.2
pycurl = 7.43.0
collective.recipe.environment = 0.2.0
smmap = 0.9.0
lockfile = 0.12.2
......@@ -13,9 +13,13 @@
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-theia]
filename = instance-theia.cfg.jinja.in
md5sum = 65c66a4cc4eb1d074dcafddf945a34d4
[instance]
filename = instance.cfg.in
md5sum = bfd488ba023f505be25d947ec830bab3
md5sum = 43923e3e1f27c43696ecbca9ee147bdb
[yarn.lock]
filename = yarn.lock
......
{% set additional_frontend = slapconfiguration_section['configuration.additional-frontend-guid'] %}
[buildout]
extends =
${monitor-template:rendered}
theia-environment-parts =
tasks.json
slapos-repository
runner-link
settings.json
request-script-template
theia-parts =
frontend-reload
promises
parts =
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
publish-connection-parameter
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[instance-parameter]
{% for k, v in slapconfiguration_section.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[publish-connection-parameter]
<= monitor-publish
recipe = slapos.cookbook:publish
url = $${remote-frontend:connection-secure_access}
{% if additional_frontend %}
additional-url = $${remote-additional-frontend:connection-secure_access}
{% endif %}
username = $${frontend-instance-password:username}
password = $${frontend-instance-password:passwd}
backend-url = $${frontend-instance:url}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
dot-theia = $${buildout:directory}/.theia/
pidfiles = $${:var}/run
services = $${:etc}/service
runner = $${:srv}/runner
project = $${:srv}/project
frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/
# Promises
# --------
[promises]
recipe =
instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${remote-frontend-url-available-promise:name}
{% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name}
{% endif %}
$${slapos-standalone-listen-promise:name}
$${slapos-autorun-promise:name}
[theia-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${theia-instance:ip}
config-port = $${theia-instance:port}
[frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[remote-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${remote-frontend:connection-secure_access}
config-check-secure = 1
{% if additional_frontend %}
[remote-additional-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${remote-additional-frontend:connection-secure_access}
config-check-secure = 1
{% endif %}
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-autorun-promise]
<= monitor-promise-base
module = check_service_state
# XXX promise plugins can not contain "slapos" in their names
name = autorun-state-promise.py
config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun}
# Remote Caddy Frontend
# ---------------------
[remote-frontend-base]
<= slap-connection
recipe = slapos.cookbook:requestoptional
slave = true
config-url = $${frontend-instance:url}
config-https-only = true
config-type = websocket
config-websocket-path-list = /services /file-upload
return = domain secure_access
[remote-frontend]
<= remote-frontend-base
name = $${instance-parameter:configuration.frontend-name}
software-url = $${instance-parameter:configuration.frontend-sr}
software-type = $${instance-parameter:configuration.frontend-sr-type}
sla-instance_guid = $${instance-parameter:configuration.frontend-guid}
{% if additional_frontend %}
[remote-additional-frontend]
<= remote-frontend-base
name = $${instance-parameter:configuration.additional-frontend-name}
software-url = $${instance-parameter:configuration.additional-frontend-sr}
software-type = $${instance-parameter:configuration.additional-frontend-sr-type}
sla-instance_guid = $${instance-parameter:configuration.additional-frontend-guid}
{% endif %}
# Local Caddy Frontend
# --------------------
[frontend-instance-password]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[frontend-instance-port]
recipe = slapos.cookbook:free_port
minimum = 3000
maximum = 3100
ip = $${instance-parameter:ipv6-random}
[frontend-instance-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:key-file} ]
then
${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:1024 -keyout $${:key-file} \
-out $${:cert-file}
fi
update-command = $${:command}
key-file = $${directory:etc}/$${:_buildout_section_name_}.key
cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
common-name = $${frontend-instance-config:ip}
location =
$${:key-file}
$${:cert-file}
[frontend-instance-config]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
:$${:port} {
bind $${:ip}
tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file}
log stdout
errors stderr
gzip
# because caddy does not support upgrade http2 to websocket
# https://tools.ietf.org/html/rfc8441
tls {
alpn http/1.1
}
root $${directory:frontend-static}
browse
proxy / $${theia-instance:base-url} {
except $${frontend-instance-fonts:folder-name} $${frontend-instance-slapos.css:folder-name} public $${favicon.ico:filename} $${frontend-instance-logo:filename}
}
proxy /services $${theia-instance:base-url} {
websocket
}
proxy /file-upload $${theia-instance:base-url} {
websocket
}
basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
realm "Theia"
/
}
}
ip = $${frontend-instance-port:ip}
hostname = [$${:ip}]
port = $${frontend-instance-port:port}
[frontend-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${caddy:output} -conf $${frontend-instance-config:rendered} -pidfile $${:pidfile}
ip = $${frontend-instance-config:ip}
hostname = $${frontend-instance-config:hostname}
port = $${frontend-instance-config:port}
pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
url = https://$${:hostname}:$${:port}/
[frontend-instance-fonts]
; XXX caddy 1 does not seem to serve different folders at different locations
; so we link fonts in static folder
recipe = plone.recipe.command
location = $${directory:frontend-static}/$${:folder-name}
folder-name = fonts
command =
mkdir -p $${:location}
ln -sf ${source-code-pro-fonts:location} $${:location}/source-code-pro
ln -sf ${jetbrains-mono-fonts:location} $${:location}/jetbrains-mono
stop-on-error = true
[frontend-instance-logo]
recipe = plone.recipe.command
filename = logo.png
full-path = $${directory:frontend-static}/$${:filename}
command =
cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true
[frontend-instance-slapos.css]
recipe = slapos.recipe.template:jinja2
template = ${slapos.css.in:output}
rendered = $${directory:frontend-static}/$${:folder-name}/slapos.css
folder-name = css
context =
key logo_image frontend-instance-logo:filename
[frontend-reload]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${bash:location}/bin/bash -c
"kill -s USR1 $$(${coreutils:location}/bin/cat $${frontend-instance:pidfile}) \
&& ${coreutils:location}/bin/sleep infinity"
hash-files =
$${frontend-instance-config:rendered}
$${frontend-instance:wrapper-path}
wait-for-files = $${frontend-instance:pidfile}
[favicon.ico]
# generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build
install =
import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false'
try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
'''$${slap-configuration:root-instance-title}'''
).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception:
# Because installation should work offline, if we can't download a favicon,
# just ignore this step.
self.logger.exception("Error while downloading favicon, using empty one")
open('''$${:location}''', 'w').close()
finally:
self.buildout['buildout']['offline'] = buildout_offline
location = $${directory:frontend-static}/$${:filename}
filename = $${:_buildout_section_name_}
# Common Environment
# ------------------
[common-environment]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:
#!/bin/sh
. ${gowork:env.sh}
# reset PS1 from gowork
export PS1='$ '
export HOME=$${buildout:directory}
export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
# Theia Backend
# -------------
[theia-service-port]
recipe = slapos.cookbook:free_port
minimum = 3500
maximum = 3600
ip = $${instance-parameter:ipv4-random}
[theia-service]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
{{ "{% raw %}" }}
{% raw %}
export THEIA_WEBVIEW_EXTERNAL_ENDPOINT='{{hostname}}'
{% endraw %}
{{ "{% endraw %}" }}
export THEIA_OPEN_EDITOR_TOKEN=$(${openssl:location}/bin/openssl rand -hex 32)
export THEIA_URL=$${:base-url}
export THEIA_SHELL=$${theia-shell:rendered}
export TMP=$${directory:tmp}
export TEMP=$TMP
export LC_ALL=C.UTF-8
export TERMINFO=${ncurses:location}/lib/terminfo/
export EDITOR="${theia-open:rendered} --wait"
export THEIA_DEFAULT_PLUGINS="local-dir:${theia-plugins:location}"
. $${common-environment:rendered}
exec ${theia-wrapper:rendered} "$@"
ip = $${theia-service-port:ip}
port = $${theia-service-port:port}
base-url = http://$${:ip}:$${:port}/
[theia-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${theia-service:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
hash-existing-files =
${yarn.lock:output}
${theia-wrapper:rendered}
ip = $${instance-parameter:ipv4-random}
hostname = $${:ip}
port = $${theia-service:port}
base-url = $${theia-service:base-url}
[theia-shell]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
{% raw %}
inline:#!{{ bash }}
SHELL=$BASH
# when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
if [ $# = 0 ]; then
. {{ activate }}
unset GIT_EXEC_PATH
set -- --rcfile {{ bashrc }}
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else
sleep 1
fi
exec "$SHELL" "$@"
{% endraw %}
context =
raw bash ${bash:location}/bin/bash
key activate slapos-standalone-activate:rendered
key bashrc theia-bashrc:rendered
[theia-bashrc]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
# enable bash completion
. ${bash-completion:location}/etc/profile.d/bash_completion.sh
# source user's .bashrc
[ -f ~/.bashrc ] && . ~/.bashrc
depends =
$${shell-setup-completion:recipe}
[shell-setup-completion]
recipe = plone.recipe.command
stop-on-error = true
command =
${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
# SlapOS Standalone
# -----------------
[slapos-standalone-port]
recipe = slapos.cookbook:free_port
minimum = 4000
maximum = 4100
ip = $${instance-parameter:ipv4-random}
[slapos-standalone-config]
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = $${slapos-standalone-port:port}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template =
inline:
export PATH=${buildout:bin-directory}:$PATH
export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
. $${common-environment:rendered}
#XXX find out where the extra nodejs in theia's PATH comes from
export PATH=${nodejs:location}/bin/:$PATH
. $${slapos-standalone-activate:rendered}
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:computer-id} \
--sr='$${instance-parameter:configuration.embedded-sr}' \
--srtype='$${instance-parameter:configuration.embedded-sr-type}' \
$${slap-connection:server-url} \
$${slap-connection:computer-id} \
$${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}'
[slapos-standalone-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
# Slapos Standalone Autoprocessing
# --------------------------------
[slapos-autorun]
recipe = plone.recipe.command
command =
case $${:autorun} in
( running ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} start $${:service-name};;
( stopped ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} stop $${:service-name};;
esac
update-command = $${:command}
service-name = slapos-node-auto
supervisor-conf = $${directory:runner}/etc/supervisord.conf
autorun = $${instance-parameter:configuration.autorun}
# Theia Local Environment Setup
# -----------------------------
[tasks.json]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:dot-theia}/tasks.json
template =
inline:
{
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
"version": "2.0.0",
"tasks": [
{
"label": "slapos node software",
"detail": "Build all software supplied to the node",
"type": "shell",
"command": "${buildout:bin-directory}/slapos",
"args": [
"node",
"software",
// debug mode can be enabled by commenting out this line:
// "--buildout-debug",
"--all"
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": []
},
{
"label": "slapos node instance",
"detail": "Create all instances requested on the node",
"type": "shell",
"command": "${buildout:bin-directory}/slapos",
"args": [
"node",
"instance",
// debug mode can be enabled by commenting out this line:
// "--buildout-debug",
"--all"
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
"problemMatcher": [],
"group": {
"kind": "build",
"isDefault": true
}
}
]
}
[slapos-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
location = $${directory:project}/slapos
branch = 1.0
develop = true
git-executable = ${git:location}/bin/git
[settings.json]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:dot-theia}$${:_buildout_section_name_}
once = $${:rendered}
template =
inline:
{
"files.watcherExclude": {
"**/.eggs/**": true,
"**/.env/**": true,
"**/.git/**": true,
"**/node_modules/**": true,
"$${directory:runner}/**":true,
"$${directory:project}/runner/**":true
}
}
[runner-link]
recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script-template]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700
template =
inline:#!/bin/sh
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
[buildout]
extends =
${monitor-template:rendered}
theia-environment-parts =
tasks.json
slapos-repository
runner-link
settings.json
request-script-template
theia-parts =
frontend-reload
promises
parts =
monitor-base
$${:theia-parts}
$${:theia-environment-parts}
publish-connection-parameter
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[instance-parameter]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
;Theia default configuration
configuration.autorun = running
configuration.embedded-sr =
configuration.embedded-sr-type =
[publish-connection-parameter]
<= monitor-publish
recipe = slapos.cookbook:publish
url = $${apache-frontend:connection-secure_access}
username = $${frontend-instance-password:username}
password = $${frontend-instance-password:passwd}
backend-url = $${frontend-instance:url}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
dot-theia = $${buildout:directory}/.theia/
pidfiles = $${:var}/run
services = $${:etc}/service
runner = $${:srv}/runner
project = $${:srv}/project
frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/
# Promises
# --------
[promises]
recipe =
instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${apache-frontend-url-available-promise:name}
$${slapos-standalone-listen-promise:name}
$${slapos-autorun-promise:name}
[theia-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${theia-instance:ip}
config-port = $${theia-instance:port}
[frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[apache-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${apache-frontend:connection-secure_access}
config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-autorun-promise]
<= monitor-promise-base
module = check_service_state
# XXX promise plugins can not contain "slapos" in their names
name = autorun-state-promise.py
config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun}
# Remote Apache Frontend
# ----------------------
[apache-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Theia Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = $${frontend-instance:url}
config-https-only = true
config-type = websocket
config-websocket-path-list = /services /file-upload
return = domain secure_access
# Local Caddy Frontend
# --------------------
[frontend-instance-password]
recipe = slapos.cookbook:generate.password
username = admin
bytes = 12
[frontend-instance-port]
recipe = slapos.cookbook:free_port
minimum = 3000
maximum = 3100
ip = $${instance-parameter:ipv6-random}
[frontend-instance-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:key-file} ]
then
${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:1024 -keyout $${:key-file} \
-out $${:cert-file}
fi
update-command = $${:command}
key-file = $${directory:etc}/$${:_buildout_section_name_}.key
cert-file = $${directory:etc}/$${:_buildout_section_name_}.crt
common-name = $${frontend-instance-config:ip}
location =
$${:key-file}
$${:cert-file}
[frontend-instance-config]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
:$${:port} {
bind $${:ip}
tls $${frontend-instance-certificate:cert-file} $${frontend-instance-certificate:key-file}
log stdout
errors stderr
gzip
# because caddy does not support upgrade http2 to websocket
# https://tools.ietf.org/html/rfc8441
tls {
alpn http/1.1
}
root $${directory:frontend-static}
browse
proxy / $${theia-instance:base-url} {
except $${frontend-instance-fonts:folder-name} $${frontend-instance-slapos.css:folder-name} public $${favicon.ico:filename} $${frontend-instance-logo:filename}
}
proxy /services $${theia-instance:base-url} {
websocket
}
proxy /file-upload $${theia-instance:base-url} {
websocket
}
basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
realm "Theia"
/
}
}
ip = $${frontend-instance-port:ip}
hostname = [$${:ip}]
port = $${frontend-instance-port:port}
[frontend-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${caddy:output} -conf $${frontend-instance-config:rendered} -pidfile $${:pidfile}
ip = $${frontend-instance-config:ip}
hostname = $${frontend-instance-config:hostname}
port = $${frontend-instance-config:port}
pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
url = https://$${:hostname}:$${:port}/
[frontend-instance-fonts]
; XXX caddy 1 does not seem to serve different folders at different locations
; so we link fonts in static folder
recipe = plone.recipe.command
location = $${directory:frontend-static}/$${:folder-name}
folder-name = fonts
command =
mkdir -p $${:location}
ln -sf ${source-code-pro-fonts:location} $${:location}/source-code-pro
ln -sf ${jetbrains-mono-fonts:location} $${:location}/jetbrains-mono
stop-on-error = true
[frontend-instance-logo]
recipe = plone.recipe.command
filename = logo.png
full-path = $${directory:frontend-static}/$${:filename}
command =
cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true
[frontend-instance-slapos.css]
recipe = slapos.recipe.template:jinja2
template = ${slapos.css.in:output}
rendered = $${directory:frontend-static}/$${:folder-name}/slapos.css
folder-name = css
;Frontend default configuration
configuration.frontend-name = Theia Frontend
configuration.frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
configuration.frontend-sr-type = RootSoftwareInstance
configuration.frontend-guid =
;Additional frontend default configuration
configuration.additional-frontend-name = Theia Additional Frontend
configuration.additional-frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
configuration.additional-frontend-sr-type = RootSoftwareInstance
configuration.additional-frontend-guid =
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = $${:theia}
theia = theia:rendered
[theia]
recipe = slapos.recipe.template:jinja2
template = ${instance-theia:output}
rendered = $${buildout:directory}/instance-theia.cfg
mode = 0644
context =
key logo_image frontend-instance-logo:filename
[frontend-reload]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${bash:location}/bin/bash -c
"kill -s USR1 $$(${coreutils:location}/bin/cat $${frontend-instance:pidfile}) \
&& ${coreutils:location}/bin/sleep infinity"
hash-files =
$${frontend-instance-config:rendered}
$${frontend-instance:wrapper-path}
wait-for-files = $${frontend-instance:pidfile}
[favicon.ico]
# generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build
install =
import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false'
try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
'''$${slap-configuration:root-instance-title}'''
).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception:
# Because installation should work offline, if we can't download a favicon,
# just ignore this step.
self.logger.exception("Error while downloading favicon, using empty one")
open('''$${:location}''', 'w').close()
finally:
self.buildout['buildout']['offline'] = buildout_offline
location = $${directory:frontend-static}/$${:filename}
filename = $${:_buildout_section_name_}
# Common Environment
# ------------------
[common-environment]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:
#!/bin/sh
. ${gowork:env.sh}
# reset PS1 from gowork
export PS1='$ '
export HOME=$${buildout:directory}
export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
# Theia Backend
# -------------
[theia-service-port]
recipe = slapos.cookbook:free_port
minimum = 3500
maximum = 3600
ip = $${instance-parameter:ipv4-random}
[theia-service]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
{% raw %}
export THEIA_WEBVIEW_EXTERNAL_ENDPOINT='{{hostname}}'
{% endraw %}
export THEIA_OPEN_EDITOR_TOKEN=$(${openssl:location}/bin/openssl rand -hex 32)
export THEIA_URL=$${:base-url}
export THEIA_SHELL=$${theia-shell:rendered}
export TMP=$${directory:tmp}
export TEMP=$TMP
export LC_ALL=C.UTF-8
export TERMINFO=${ncurses:location}/lib/terminfo/
export EDITOR="${theia-open:rendered} --wait"
export THEIA_DEFAULT_PLUGINS="local-dir:${theia-plugins:location}"
. $${common-environment:rendered}
exec ${theia-wrapper:rendered} "$@"
ip = $${theia-service-port:ip}
port = $${theia-service-port:port}
base-url = http://$${:ip}:$${:port}/
[theia-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${theia-service:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
hash-existing-files =
${yarn.lock:output}
${theia-wrapper:rendered}
ip = $${instance-parameter:ipv4-random}
hostname = $${:ip}
port = $${theia-service:port}
base-url = $${theia-service:base-url}
[theia-shell]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!{{ bash }}
SHELL=$BASH
# when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
if [ $# = 0 ]; then
. {{ activate }}
unset GIT_EXEC_PATH
set -- --rcfile {{ bashrc }}
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else
sleep 1
fi
exec "$SHELL" "$@"
context =
raw bash ${bash:location}/bin/bash
key activate slapos-standalone-activate:rendered
key bashrc theia-bashrc:rendered
[theia-bashrc]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
# enable bash completion
. ${bash-completion:location}/etc/profile.d/bash_completion.sh
# source user's .bashrc
[ -f ~/.bashrc ] && . ~/.bashrc
depends =
$${shell-setup-completion:recipe}
[shell-setup-completion]
recipe = plone.recipe.command
stop-on-error = true
command =
${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
# SlapOS Standalone
# -----------------
[slapos-standalone-port]
recipe = slapos.cookbook:free_port
minimum = 4000
maximum = 4100
ip = $${instance-parameter:ipv4-random}
[slapos-standalone-config]
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = $${slapos-standalone-port:port}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template =
inline:
export PATH=${buildout:bin-directory}:$PATH
export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
. $${common-environment:rendered}
#XXX find out where the extra nodejs in theia's PATH comes from
export PATH=${nodejs:location}/bin/:$PATH
. $${slapos-standalone-activate:rendered}
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:computer-id} \
--sr='$${instance-parameter:configuration.embedded-sr}' \
--srtype='$${instance-parameter:configuration.embedded-sr-type}' \
$${slap-connection:server-url} \
$${slap-connection:computer-id} \
$${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}'
[slapos-standalone-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
# Slapos Standalone Autoprocessing
# --------------------------------
[slapos-autorun]
recipe = plone.recipe.command
command =
case $${:autorun} in
( running ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} start $${:service-name};;
( stopped ) ${buildout:bin-directory}/supervisorctl -c $${:supervisor-conf} stop $${:service-name};;
esac
update-command = $${:command}
service-name = slapos-node-auto
supervisor-conf = $${directory:runner}/etc/supervisord.conf
autorun = $${instance-parameter:configuration.autorun}
# Theia Local Environment Setup
# -----------------------------
[tasks.json]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:dot-theia}/tasks.json
template =
inline:
{
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
"version": "2.0.0",
"tasks": [
{
"label": "slapos node software",
"detail": "Build all software supplied to the node",
"type": "shell",
"command": "${buildout:bin-directory}/slapos",
"args": [
"node",
"software",
// debug mode can be enabled by commenting out this line:
// "--buildout-debug",
"--all"
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": []
},
{
"label": "slapos node instance",
"detail": "Create all instances requested on the node",
"type": "shell",
"command": "${buildout:bin-directory}/slapos",
"args": [
"node",
"instance",
// debug mode can be enabled by commenting out this line:
// "--buildout-debug",
"--all"
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
"problemMatcher": [],
"group": {
"kind": "build",
"isDefault": true
}
}
]
}
[slapos-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
location = $${directory:project}/slapos
branch = 1.0
develop = true
git-executable = ${git:location}/bin/git
[settings.json]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:dot-theia}$${:_buildout_section_name_}
once = $${:rendered}
template =
inline:
{
"files.watcherExclude": {
"**/.eggs/**": true,
"**/.env/**": true,
"**/.git/**": true,
"**/node_modules/**": true,
"$${directory:runner}/**":true,
"$${directory:project}/runner/**":true
}
}
[runner-link]
recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script-template]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700
template =
inline:#!/bin/sh
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
section slapconfiguration_section slap-configuration
......@@ -22,6 +22,7 @@ extends =
parts =
theia-wrapper
slapos-cookbook
instance-theia
instance
# default for slapos-standalone
......@@ -323,6 +324,10 @@ template =
#!/bin/sh
exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
[instance-theia]
<= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja
[instance]
<= template-base
output = ${buildout:directory}/instance.cfg
......@@ -247,6 +247,22 @@ class TestTheiaWithSR(TheiaTestCase):
self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (self.sr_url, self.sr_type, instance_name), info), info)
class TestTheiaFrontend(TheiaTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'additional-frontend-guid': 'SOMETHING'
}
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def test_http_get(self):
for key in ('url', 'additional-url'):
resp = requests.get(self.connection_parameters[key], verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
class TestTheiaEnv(TheiaTestCase):
dummy_software_path = os.path.abspath('dummy/software.cfg')
......
......@@ -38,5 +38,5 @@ cuda_toolkit_path = /usr/local/cuda
cudnn_install_path = /usr/local/cuda
[versions]
mock = 2.0.0
mock = 2.0.0:whl
funcsigs = 1.0.2
......@@ -71,7 +71,7 @@ command = true
[versions]
Paste = 3.4.0
PasteScript = 3.2.0
PasteScript = 3.2.0:whl
WSGIUtils = 0.7.2
WSGIserver = 1.3
python-magic = 0.4.18
......
......@@ -25,8 +25,6 @@ extends =
../component/lxml-python/buildout.cfg
../component/python-backports-lzma/buildout.cfg
../component/python-cffi/buildout.cfg
../component/python-cliff/buildout.cfg
../component/python-cachecontrol/buildout.cfg
../component/python-cryptography/buildout.cfg
../component/python-PyYAML/buildout.cfg
../component/pycurl/buildout.cfg
......@@ -103,11 +101,8 @@ eggs =
${lxml-python:egg}
${python-cffi:egg}
${python-PyYAML:egg}
${python-cachecontrol:egg}
${python-cliff:egg}
${python-cryptography:egg}
${pyrsistent:egg}
${jsonschema:egg}
# slapos.toolbox containing utilities
[slapos-toolbox]
......@@ -121,36 +116,23 @@ depends = ${slapos-toolbox-dependencies:eggs}
# slapos.toolbox dependencies.
recipe = zc.recipe.egg
eggs =
${python-cliff:egg}
${lxml-python:egg}
${pycurl:egg}
${python-cryptography:egg}
${backports.lzma:egg}
${python-dateutil:egg}
[jsonschema]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = setuptools_scm
[python-dateutil]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = setuptools_scm
# Install a slapos command with networkcache enabled in ${buildout:bin-directory}
[slapos-command]
recipe = zc.recipe.egg
eggs =
${python-cliff:egg}
${lxml-python:egg}
slapos.core
slapos.libnetworkcache
[versions]
setuptools = 44.0.0
setuptools = 44.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.7.1+slapos010
zc.buildout = 2.7.1+slapos013
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -165,11 +147,11 @@ Werkzeug = 1.0.1
ZConfig = 2.9.3
asn1crypto = 1.3.0
atomicwrites = 1.4.0
backports.functools-lru-cache = 1.6.1
backports.functools-lru-cache = 1.6.1:whl
backports.lzma = 0.0.14
cffi = 1.14.0
click = 6.7
cliff = 2.4.0
cliff = 2.8.3:whl
cmd2 = 0.7.0
collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0
......@@ -217,7 +199,7 @@ slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5
slapos.toolbox = 0.119
stevedore = 1.21.0
stevedore = 1.21.0:whl
subprocess32 = 3.5.4
unicodecsv = 0.14.1
wcwidth = 0.2.5
......@@ -229,7 +211,7 @@ zipp = 1.2.0
zodburi = 2.4.0
zope.event = 3.5.2
paramiko = 2.1.3
CacheControl = 0.12.5
CacheControl = 0.12.6:whl
msgpack = 0.6.2
# Required by:
......@@ -290,21 +272,17 @@ attrs = 18.2.0
# jsonschema==3.0.2
pyrsistent = 0.16.1
pytest-runner = 5.2.0
# Required by:
# jsonschema==3.0.2
setuptools-scm = 3.5.0
pytest-runner = 5.2.0:whl
ipaddress = 1.0.23
# Required by:
# slapos.cookbook==1.0.143
jsonschema = 3.0.2
jsonschema = 3.0.2:whl
# Required by:
# slapos.toolbox==0.94
lockfile = 0.12.2
lockfile = 0.12.2:whl
# Required by:
# slapos.core==1.5.0
......@@ -331,7 +309,7 @@ pycurl = 7.43.0
# Required by:
# slapos.toolbox==0.94
python-dateutil = 2.7.3
python-dateutil = 2.7.3:whl
# Required by:
# slapos.toolbox==0.94
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment