theia: include a slapos

Include a slapos command implemented with slapos standalone

See merge request !743

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 0edf9e67bc637d73a7415124c8082a8c
+md5sum = 13f4f3806522b265dd1912ff169d146d
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in

software/grafana/instance.cfg.in

@@ -267,6 +267,10 @@ config-url = ${grafana:url}
 config-https-only = true
 return = domain secure_access
 
+[apache-frontend-available-promise]
+<= check-url-available-promise
+url = ${apache-frontend:connection-secure_access}
+
 
 [promises]
 recipe =
@@ -277,6 +281,8 @@ instance-promises =
   ${grafana-listen-promise:path}
   ${loki-listen-promise:path}
   ${promtail-listen-promise:path}
+  ${promtail-listen-promise:path}
+  ${apache-frontend-available-promise:path}
 
 
 [publish-connection-parameter]

software/slapos-sr-testing/software.cfg

@@ -291,4 +291,6 @@ mock = 2.0.0
 testfixtures = 6.11
 funcsigs = 1.0.2
 PyPDF2 = 1.26.0
-mysqlclient = 1.3.12
+mysqlclient = 1.3.12
\ No newline at end of file
+pexpect = 4.8.0
+ptyprocess = 0.6.0

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 21735765808aac82fb91d53341a3c0d6
+md5sum = 99dfaa031e58518326b731e9218aa425
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = c2523a5c832f617c374ee621d50d9e52
+md5sum = ae1b596804715acd3512f1e8e6cbae3b
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 6db2a484cac19787fecd87fffefa4aa9
+md5sum = 2a296cba4c36d7a5fca5f4347a8c8469

software/theia/instance.cfg.in

@@ -50,8 +50,10 @@ template = inline:
     tls {
       alpn http/1.1
     }
+    root $${directory:frontend-static}
+    browse
     proxy / $${theia-instance:base-url} {
-    #  transparent
+      except public $${favicon.ico:filename}
     }
     proxy /services $${theia-instance:base-url} {
       websocket
@@ -89,6 +91,29 @@ hash-files =
   $${frontend-instance:wrapper-path}
 wait-for-files = $${frontend-instance:pidfile}
 
+[favicon.ico]
+# generate a pseudo random favicon, different for each instance name.
+recipe = slapos.recipe.build
+install =
+  import hashlib, shutil
+  buildout_offline = self.buildout['buildout']['offline']
+  self.buildout['buildout']['offline'] = 'false'
+  try:
+    gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
+      '''$${slap-configuration:root-instance-title}'''
+    ).hexdigest() + "?s=256&d=retro"
+    shutil.copy(self.download(gravatar_url), '''$${:location}''')
+  except Exception:
+    # Because installation should work offline, if we can't download a favicon,
+    # just ignore this step.
+    self.logger.exception("Error while downloading favicon, using empty one")
+    open('''$${:location}''', 'w').close()
+  finally:
+    self.buildout['buildout']['offline'] = buildout_offline
+
+location = $${directory:frontend-static}/$${:filename}
+filename = $${:_buildout_section_name_}
+
 [user]
 recipe = slapos.cookbook:userinfo
 
@@ -96,7 +121,7 @@ recipe = slapos.cookbook:userinfo
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
+  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} THEIA_SHELL=$${theia-shell:wrapper-path} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
 
 ip =  $${instance-parameter:ipv4-random}
 hostname =  $${:ip}
@@ -106,11 +131,37 @@ hash-existing-files =
   ${yarn.lock:output}
   ${theia-wrapper:rendered}
 
+[theia-shell]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+# reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
+# activate slapos configuration
+command-line =
+  ${bash:location}/bin/bash -c ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash"
+
+[slapos-standalone-activate]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+mode = 0700
+# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
+template =
+  inline:#!/bin/sh
+  export PATH=${buildout:bin-directory}:$PATH
+  ${slapos-standalone:script-path} $${directory:slapos} $${:ipv4} $${:ipv6} $${:port}
+  export SLAPOS_CONFIGURATION=$${directory:slapos}/etc/slapos.cfg
+  export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
+
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+port = 4000
+
+
 [promises]
 recipe =
 instance-promises =
   $${theia-listen-promise:name}
   $${frontend-listen-promise:name}
+  $${apache-frontend-url-available-promise:name}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -126,6 +177,13 @@ name = $${:_buildout_section_name_}.py
 config-hostname = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
+[apache-frontend-url-available-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${apache-frontend:connection-secure_access}
+config-check-secure = 1
+
 [apache-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -163,6 +221,7 @@ tmp = $${buildout:directory}/tmp
 pidfiles = $${:var}/run
 
 services = $${:etc}/service
-framebuffer = $${:srv}/framebuffer
-fonts = $${:srv}/fonts
-home = $${:srv}/home
+project = $${:srv}/project
+slapos = $${:srv}/slapos
+frontend-static = $${:srv}/frontend-static
+frontend-static-public = $${:frontend-static}/public

software/theia/python-language-server-requirements.txt

@@ -26,4 +26,4 @@ typed-ast==1.4.1
 typing-extensions==3.7.4.2
 wrapt==1.11.2
 yapf==0.29.0
-zc.buildout.languageserver==0.2.0
+zc.buildout.languageserver==0.2.1

software/theia/software.cfg

@@ -4,6 +4,11 @@ extends =
    ../../component/caddy/buildout.cfg
    ../../component/git/buildout.cfg
    ../../component/bash/buildout.cfg
+   ../../component/fish-shell/buildout.cfg
+   ../../component/tmux/buildout.cfg
+   ../../component/tig/buildout.cfg
+   ../../component/vim/buildout.cfg
+   ../../component/curl/buildout.cfg
    ../../component/coreutils/buildout.cfg
    ../../stack/slapos.cfg
    ../../stack/monitor/buildout.cfg
@@ -20,6 +25,9 @@ parts =
   slapos-cookbook
   instance
 
+# default for slapos-standalone
+shared-part-list =
+
 
 [nodejs]
 <= nodejs-10.19.0
@@ -33,12 +41,66 @@ recipe = slapos.recipe.build:download-unpacked
 url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
 md5sum = 4a02e1687a150113ad6b0215f9afdb3e
 
+[slapos-standalone]
+recipe = zc.recipe.egg
+eggs =
+  slapos.core
+scripts = ${:_buildout_section_name_}
+script-path = ${buildout:bin-directory}/${:scripts}
+# XXX generate a fake entry point for a non existant module, that will not
+# be used because we exit in initialization step
+entry-points =
+  ${:scripts}=not_used:main
+initialization =
+  import argparse
+  import os.path
+  import sys
+  import glob
+
+  import slapos.slap.standalone
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('base_directory')
+  parser.add_argument('ipv4')
+  parser.add_argument('ipv6')
+  parser.add_argument('server_port', type=int)
+  args = parser.parse_args()
+  shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
+
+  standalone = slapos.slap.standalone.StandaloneSlapOS(
+      args.base_directory,
+      args.ipv4,
+      args.server_port,
+      shared_part_list=shared_part_list
+  )
+  standalone.start()
+  partition_count = 20
+  if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
+    print("Standalone SlapOS: Formatting {partition_count} partitions".format(
+        partition_count=partition_count))
+    standalone.format(
+        partition_count,
+        args.ipv4,
+        args.ipv6
+    )
+  print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
+  sys.exit(0)
+
+needs-these-eggs-scripts-in-path =
+  ${supervisor:recipe}
+  ${slapos-command:recipe}
+
+[supervisor]
+recipe = zc.recipe.egg
+eggs =
+  supervisor
+  setuptools
 
 [python-language-server]
 version = 0.19.0
 recipe = plone.recipe.command
 command =
-  bash -c "${python3:executable} -m venv ${:location} && \
+  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv ${:location} && \
     . ${:location}/bin/activate && \
     pip install -r ${python-language-server-requirements.txt:output}"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -77,9 +139,7 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/0cc6a3dc2a6dec1b2b4f9572e9d878a67dc81ff5/theia-full-docker/latest.package.json
-# but we pin vscode-json-languageserver to 1.2.2, otherwise 1.2.3 is picked and it seems to be incompatible.*
-# See https://github.com/theia-ide/theia-apps/issues/333
+# this comes from https://github.com/theia-ide/theia-apps/blob/2991e3a433f031b22bc80e274f80620d1e6898e5/theia-full-docker/latest.package.json
 template =
   inline:{
       "private": true,
@@ -145,6 +205,7 @@ template =
           "@theia/terminal": "latest",
           "@theia/userstorage": "latest",
           "@theia/variable-resolver": "latest",
+          "@theia/vsx-registry": "latest",
           "@theia/workspace": "latest"
       },
       "resolutions": {
@@ -226,7 +287,7 @@ template =
           "vscode-java-dependency-viewer": "https://github.com/microsoft/vscode-java-dependency/releases/download/0.6.0/vscode-java-dependency-0.6.0.vsix",
           "vscode-java-redhat": "https://github.com/redhat-developer/vscode-java/releases/download/v0.54.2/redhat.java-0.54.2.vsix",
           "vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
-          "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix",
+          "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.4.74986/ms-python-release.vsix",
           "vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
           "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix"
       }
@@ -257,6 +318,8 @@ install +=
   github.com/haya14busa/goplay/cmd/goplay
   github.com/davidrjenni/reftools/cmd/fillstruct
 
+[cli-utilities]
+PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin
 
 [theia-wrapper]
 recipe = slapos.recipe.template:jinja2
@@ -265,10 +328,9 @@ mode = 0777
 template =
   inline:
   #!/bin/bash
-  export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
   . ${gowork:env.sh}
+  export PATH=${python-language-server:location}/bin/:${cli-utilities:PATH}:$PATH
   export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}"
-  export SHELL=bash
   # reset PS1 from gowork
   export PS1='$ '
   cd ${theia:location}

software/theia/test/setup.py

@@ -45,6 +45,7 @@ setup(
         'slapos.libnetworkcache',
         'erp5.util',
         'supervisor',
+        'pexpect',
         'requests',
     ],
     zip_safe=True,

software/theia/test/test.py

@@ -24,25 +24,27 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import unicode_literals
 
 import os
 import textwrap
 import logging
 import tempfile
 import time
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, urljoin
 
+import pexpect
 import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
-
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
 class TestTheia(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for sockets in included slapos
   def setUp(self):
     self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
@@ -52,12 +54,71 @@ class TestTheia(SlapOSInstanceTestCase):
 
     # with login/password, this is allowed
     parsed_url = urlparse(self.connection_parameters['url'])
+    authenticated_url = parsed_url._replace(
+        netloc='{}:{}@[{}]:{}'.format(
+            self.connection_parameters['username'],
+            self.connection_parameters['password'],
+            parsed_url.hostname,
+            parsed_url.port,
+        )).geturl()
+    resp = requests.get(authenticated_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+    # there's a public folder to serve file
+    with open('{}/srv/frontend-static/public/test_file'.format(
+        self.computer_partition_root_path), 'w') as f:
+      f.write("hello")
+    resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
+    self.assertIn('test_file', resp.text)
     resp = requests.get(
-        parsed_url._replace(
-            netloc='{}:{}@[{}]:{}'.format(
-                self.connection_parameters['username'],
-                self.connection_parameters['password'],
-                parsed_url.hostname,
-                parsed_url.port)).geturl(),
-        verify=False)
+        urljoin(authenticated_url, '/public/test_file'), verify=False)
+    self.assertEqual('hello', resp.text)
+
+    # there's a (not empty) favicon
+    resp = requests.get(
+        urljoin(authenticated_url, '/favicon.ico'), verify=False)
     self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertTrue(resp.raw)
+
+  def test_theia_slapos(self):
+    # Make sure we can use the shell and the integrated slapos command
+    process = pexpect.spawnu(
+        '{}/bin/theia-shell'.format(self.computer_partition_root_path),
+        env={'HOME': self.computer_partition_root_path})
+
+    # use a large enough terminal so that slapos proxy show table fit in the screen
+    process.setwinsize(5000, 5000)
+
+    process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
+    process.expect_exact('Standalone SlapOS for computer `local` activated')
+
+    # try to supply and install a software to check that this slapos is usable
+    process.sendline(
+        'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local'
+    )
+    process.expect(
+        'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
+    )
+
+    # we pipe through cat to disable pager and prevent warnings like
+    # WARNING: terminal is not fully functional
+    process.sendline('slapos proxy show | cat')
+    process.expect(
+        'https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
+    )
+
+    process.sendline('slapos node software')
+    process.expect(
+        'Installing software release https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
+    )
+    # interrupt this, we don't want to actually wait for software installation
+    process.sendcontrol('c')
+
+    # shutdown this slapos
+    process.sendline(
+        'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
+            self.computer_partition_root_path))
+    process.expect('Shut down')
+
+    process.terminate()
+    process.wait()