Skip to content

slapos
slapos
Commit cdd6690d authored by Jérome Perrin's avatar Jérome Perrin
1 Browse files
Options

theia: include a slapos 

Include a slapos command implemented with slapos standalone

See merge request !743
parents 7bf2cf5a 358b08c5
Hide whitespace changes
Inline Side-by-side
Showing with 1489 additions and 737 deletions
software/grafana/buildout.hash.cfg
View file @ cdd6690d
......@@ -15,7 +15,7 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 0edf9e67bc637d73a7415124c8082a8c
md5sum = 13f4f3806522b265dd1912ff169d146d
[influxdb-config-file]
filename = influxdb-config-file.cfg.in
......
software/grafana/instance.cfg.in
View file @ cdd6690d
......@@ -267,6 +267,10 @@ config-url = ${grafana:url}
config-https-only = true
return = domain secure_access
[apache-frontend-available-promise]
<= check-url-available-promise
url = ${apache-frontend:connection-secure_access}
[promises]
recipe =
......@@ -277,6 +281,8 @@ instance-promises =
${grafana-listen-promise:path}
${loki-listen-promise:path}
${promtail-listen-promise:path}
${promtail-listen-promise:path}
${apache-frontend-available-promise:path}
[publish-connection-parameter]
......
software/slapos-sr-testing/software.cfg
View file @ cdd6690d
......@@ -291,4 +291,6 @@ mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
PyPDF2 = 1.26.0
mysqlclient = 1.3.12
\ No newline at end of file
mysqlclient = 1.3.12
pexpect = 4.8.0
ptyprocess = 0.6.0
software/theia/buildout.hash.cfg
View file @ cdd6690d
......@@ -15,12 +15,12 @@
[instance]
filename = instance.cfg.in
md5sum = 21735765808aac82fb91d53341a3c0d6
md5sum = 99dfaa031e58518326b731e9218aa425
[yarn.lock]
filename = yarn.lock
md5sum = c2523a5c832f617c374ee621d50d9e52
md5sum = ae1b596804715acd3512f1e8e6cbae3b
[python-language-server-requirements.txt]
filename = python-language-server-requirements.txt
md5sum = 6db2a484cac19787fecd87fffefa4aa9
md5sum = 2a296cba4c36d7a5fca5f4347a8c8469
software/theia/instance.cfg.in
View file @ cdd6690d
......@@ -50,8 +50,10 @@ template = inline:
tls {
alpn http/1.1
}
root $${directory:frontend-static}
browse
proxy / $${theia-instance:base-url} {
# transparent
except public $${favicon.ico:filename}
}
proxy /services $${theia-instance:base-url} {
websocket
......@@ -89,6 +91,29 @@ hash-files =
$${frontend-instance:wrapper-path}
wait-for-files = $${frontend-instance:pidfile}
[favicon.ico]
# generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build
install =
import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false'
try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
'''$${slap-configuration:root-instance-title}'''
).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception:
# Because installation should work offline, if we can't download a favicon,
# just ignore this step.
self.logger.exception("Error while downloading favicon, using empty one")
open('''$${:location}''', 'w').close()
finally:
self.buildout['buildout']['offline'] = buildout_offline
location = $${directory:frontend-static}/$${:filename}
filename = $${:_buildout_section_name_}
[user]
recipe = slapos.cookbook:userinfo
......@@ -96,7 +121,7 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
env LC_ALL=C.UTF-8 TMP=$${directory:tmp} THEIA_SHELL=$${theia-shell:wrapper-path} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
ip = $${instance-parameter:ipv4-random}
hostname = $${:ip}
......@@ -106,11 +131,37 @@ hash-existing-files =
${yarn.lock:output}
${theia-wrapper:rendered}
[theia-shell]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
# reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
# activate slapos configuration
command-line =
${bash:location}/bin/bash -c ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash"
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
template =
inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH
${slapos-standalone:script-path} $${directory:slapos} $${:ipv4} $${:ipv6} $${:port}
export SLAPOS_CONFIGURATION=$${directory:slapos}/etc/slapos.cfg
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 4000
[promises]
recipe =
instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${apache-frontend-url-available-promise:name}
[theia-listen-promise]
<= monitor-promise-base
......@@ -126,6 +177,13 @@ name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[apache-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${apache-frontend:connection-secure_access}
config-check-secure = 1
[apache-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
......@@ -163,6 +221,7 @@ tmp = $${buildout:directory}/tmp
pidfiles = $${:var}/run
services = $${:etc}/service
framebuffer = $${:srv}/framebuffer
fonts = $${:srv}/fonts
home = $${:srv}/home
project = $${:srv}/project
slapos = $${:srv}/slapos
frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
software/theia/python-language-server-requirements.txt
View file @ cdd6690d
......@@ -26,4 +26,4 @@ typed-ast==1.4.1
typing-extensions==3.7.4.2
wrapt==1.11.2
yapf==0.29.0
zc.buildout.languageserver==0.2.0
zc.buildout.languageserver==0.2.1
software/theia/software.cfg
View file @ cdd6690d
......@@ -4,6 +4,11 @@ extends =
../../component/caddy/buildout.cfg
../../component/git/buildout.cfg
../../component/bash/buildout.cfg
../../component/fish-shell/buildout.cfg
../../component/tmux/buildout.cfg
../../component/tig/buildout.cfg
../../component/vim/buildout.cfg
../../component/curl/buildout.cfg
../../component/coreutils/buildout.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
......@@ -20,6 +25,9 @@ parts =
slapos-cookbook
instance
# default for slapos-standalone
shared-part-list =
[nodejs]
<= nodejs-10.19.0
......@@ -33,12 +41,66 @@ recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[slapos-standalone]
recipe = zc.recipe.egg
eggs =
slapos.core
scripts = ${:_buildout_section_name_}
script-path = ${buildout:bin-directory}/${:scripts}
# XXX generate a fake entry point for a non existant module, that will not
# be used because we exit in initialization step
entry-points =
${:scripts}=not_used:main
initialization =
import argparse
import os.path
import sys
import glob
import slapos.slap.standalone
parser = argparse.ArgumentParser()
parser.add_argument('base_directory')
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
args = parser.parse_args()
shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
standalone = slapos.slap.standalone.StandaloneSlapOS(
args.base_directory,
args.ipv4,
args.server_port,
shared_part_list=shared_part_list
)
standalone.start()
partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(
partition_count,
args.ipv4,
args.ipv6
)
print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
sys.exit(0)
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
[python-language-server]
version = 0.19.0
recipe = plone.recipe.command
command =
bash -c "${python3:executable} -m venv ${:location} && \
PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
......@@ -77,9 +139,7 @@ mode = 0644
[package.json]
recipe = slapos.recipe.template:jinja2
# this comes from https://github.com/theia-ide/theia-apps/blob/0cc6a3dc2a6dec1b2b4f9572e9d878a67dc81ff5/theia-full-docker/latest.package.json
# but we pin vscode-json-languageserver to 1.2.2, otherwise 1.2.3 is picked and it seems to be incompatible.*
# See https://github.com/theia-ide/theia-apps/issues/333
# this comes from https://github.com/theia-ide/theia-apps/blob/2991e3a433f031b22bc80e274f80620d1e6898e5/theia-full-docker/latest.package.json
template =
inline:{
"private": true,
......@@ -145,6 +205,7 @@ template =
"@theia/terminal": "latest",
"@theia/userstorage": "latest",
"@theia/variable-resolver": "latest",
"@theia/vsx-registry": "latest",
"@theia/workspace": "latest"
},
"resolutions": {
......@@ -226,7 +287,7 @@ template =
"vscode-java-dependency-viewer": "https://github.com/microsoft/vscode-java-dependency/releases/download/0.6.0/vscode-java-dependency-0.6.0.vsix",
"vscode-java-redhat": "https://github.com/redhat-developer/vscode-java/releases/download/v0.54.2/redhat.java-0.54.2.vsix",
"vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
"vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix",
"vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.4.74986/ms-python-release.vsix",
"vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
"vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix"
}
......@@ -257,6 +318,8 @@ install +=
github.com/haya14busa/goplay/cmd/goplay
github.com/davidrjenni/reftools/cmd/fillstruct
[cli-utilities]
PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin
[theia-wrapper]
recipe = slapos.recipe.template:jinja2
......@@ -265,10 +328,9 @@ mode = 0777
template =
inline:
#!/bin/bash
export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
. ${gowork:env.sh}
export PATH=${python-language-server:location}/bin/:${cli-utilities:PATH}:$PATH
export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}"
export SHELL=bash
# reset PS1 from gowork
export PS1='$ '
cd ${theia:location}
......
software/theia/test/setup.py
View file @ cdd6690d
......@@ -45,6 +45,7 @@ setup(
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'pexpect',
'requests',
],
zip_safe=True,
......
software/theia/test/test.py
View file @ cdd6690d
......@@ -24,25 +24,27 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import textwrap
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestTheia(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for sockets in included slapos
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
......@@ -52,12 +54,71 @@ class TestTheia(SlapOSInstanceTestCase):
# with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['url'])
authenticated_url = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['username'],
self.connection_parameters['password'],
parsed_url.hostname,
parsed_url.port,
)).geturl()
resp = requests.get(authenticated_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
# there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f:
f.write("hello")
resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
self.assertIn('test_file', resp.text)
resp = requests.get(
parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['username'],
self.connection_parameters['password'],
parsed_url.hostname,
parsed_url.port)).geturl(),
verify=False)
urljoin(authenticated_url, '/public/test_file'), verify=False)
self.assertEqual('hello', resp.text)
# there's a (not empty) favicon
resp = requests.get(
urljoin(authenticated_url, '/favicon.ico'), verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertTrue(resp.raw)
def test_theia_slapos(self):
# Make sure we can use the shell and the integrated slapos command
process = pexpect.spawnu(
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
env={'HOME': self.computer_partition_root_path})
# use a large enough terminal so that slapos proxy show table fit in the screen
process.setwinsize(5000, 5000)
process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
process.expect_exact('Standalone SlapOS for computer `local` activated')
# try to supply and install a software to check that this slapos is usable
process.sendline(
'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local'
)
process.expect(
'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
)
# we pipe through cat to disable pager and prevent warnings like
# WARNING: terminal is not fully functional
process.sendline('slapos proxy show | cat')
process.expect(
'https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
)
process.sendline('slapos node software')
process.expect(
'Installing software release https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
)
# interrupt this, we don't want to actually wait for software installation
process.sendcontrol('c')
# shutdown this slapos
process.sendline(
'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
self.computer_partition_root_path))
process.expect('Shut down')
process.terminate()
process.wait()
software/theia/yarn.lock
View file @ cdd6690d
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7