Skip to content

slapos
slapos
Commit fc5a017f authored by Thomas Gambier's avatar Thomas Gambier
Browse files
Options

Update Release Candidate

parents d2c3e801 18f9a013
Expand all Show whitespace changes
Inline Side-by-side
Showing with 290 additions and 3699 deletions
component/caddy/buildout.cfg deleted 100644 → 0
View file @ d2c3e801
[buildout]
extends =
../../component/golang/buildout.cfg
parts =
gowork
caddy
[caddy-get]
<= go-git-package
go.importpath = github.com/caddyserver/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-03fba31bf
[gowork]
golang = ${golang1.17:location}
install =
${caddy-get:location}:./...
[caddy]
recipe = plone.recipe.command
command = exit 0
update-command = ${:command}
output = ${gowork:bin}/caddy
location = ${:output}
component/haproxy/buildout.cfg
View file @ fc5a017f
......@@ -13,8 +13,8 @@ parts = haproxy
[haproxy]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.haproxy.org/download/2.6/src/haproxy-2.6.7.tar.gz
md5sum = cfa36413f2bc5187ab34ffcdf71914d4
url = https://www.haproxy.org/download/2.6/src/haproxy-2.6.8.tar.gz
md5sum = 0b506378f1ade633d5b921f5806a5e82
configure-command = true
# for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
# otherwise use "generic".
......
software/erp5testnode/buildout.hash.cfg
View file @ fc5a017f
......@@ -18,4 +18,4 @@ md5sum = 5bd72da73a8b84f70f568686924d7d6c
[template-default]
filename = instance-default.cfg.jinja.in
md5sum = 2aafd7708aac85847da3a8679b84fe34
md5sum = 96fb2221c8a91fb44dc5b931c29e0af9
software/erp5testnode/instance-default.cfg.jinja.in
View file @ fc5a017f
......@@ -107,23 +107,43 @@ inline =
--unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
--service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
[shellinabox-frontend-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:cert-file} ]
then
${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:2048 -keyout $${:cert-file} \
-out $${:cert-file}
fi
update-command = $${:command}
cert-file = $${directory:var}/$${:_buildout_section_name_}.pem
common-name = $${:ipv6}
location =
$${:cert-file}
ipv6 = {{ partition_ipv6 }}
[shellinabox-frontend-config]
recipe = slapos.recipe.template
recipe = slapos.recipe.template:jinja2
url = ${stack-haproxy-default-backend-config:target}
output = $${directory:etc}/$${:_buildout_section_name_}
inline =
https://$${:hostname}:$${:port} {
bind $${:ipv6}
tls self_signed
gzip
log stdout
errors stderr
proxy / unix:$${shellinabox:socket}
basicauth $${:username} $${:passwd} {
realm "Test Node $${testnode:test-node-title}"
/
}
}
ipv6 = {{ partition_ipv6 }}
context =
key pidfile :pidfile
key content :content
pidfile = $${:pidfile}
content =
userlist auth
user $${:username} insecure-password $${:passwd}
listen app
log global
acl auth_ok http_auth(auth)
http-request auth realm "Test Node $${testnode:test-node-title}" unless auth_ok
bind $${:ipv6}:$${:port} ssl crt $${shellinabox-frontend-certificate:cert-file} alpn h2,http/1.1
server app unix@$${shellinabox:socket}
ipv6 = $${shellinabox-frontend-certificate:ipv6}
hostname = [$${:ipv6}]
port = 8080
username = testnode
......@@ -131,24 +151,24 @@ passwd = $${pwgen:passwd}
cert-file = $${directory:shellinabox}/public.crt
key-file = $${directory:shellinabox}/private.key
backend-url = https://$${:username}:$${:passwd}@$${:hostname}:$${:port}
pidfile = $${basedirectory:run}/shellinabox-haproxy.pid
[shellinabox-frontend]
recipe = slapos.cookbook:wrapper
wrapper-path = $${rootdirectory:bin}/$${:_buildout_section_name_}
command-line =
${caddy:output} -conf $${shellinabox-frontend-config:output} -pidfile $${:pidfile}
${haproxy:location}/sbin/haproxy -f $${shellinabox-frontend-config:output}
url = $${shellinabox-frontend-config:backend-url}
hostname = $${shellinabox-frontend-config:ipv6}
port = $${shellinabox-frontend-config:port}
pidfile = $${basedirectory:run}/$${:_buildout_section_name_}.pid
pidfile = $${shellinabox-frontend-config:pidfile}
[shellinabox-frontend-reload]
recipe = slapos.cookbook:wrapper
wrapper-path = $${basedirectory:services}/$${:_buildout_section_name_}
command-line =
${bash:location}/bin/bash -c
"kill -s USR1 $$(${coreutils:location}/bin/cat $${shellinabox-frontend:pidfile}) \
"kill -s USR2 $$(${coreutils:location}/bin/cat $${shellinabox-frontend:pidfile}) \
&& ${coreutils:location}/bin/sleep infinity"
hash-files =
$${shellinabox-frontend-config:output}
......
software/erp5testnode/software.cfg
View file @ fc5a017f
......@@ -2,11 +2,11 @@
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/haproxy/default-backend.cfg
../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/zip/buildout.cfg
../../component/bash/buildout.cfg
../../component/caddy/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/shellinabox/buildout.cfg
../../component/pwgen/buildout.cfg
......
software/erp5testnode/testsuite/slaprunner.cfg deleted 100644 → 0
View file @ d2c3e801
[buildout]
extends =
../../slaprunner/development.cfg
testsuite.cfg
parts += template-erp5testnode
[template-resilient-test]
filename = instance-resilient-test.cfg.jinja2
md5sum = e4b04aa6fd3413bc6ae38823abdcc8fa
# We have to use an extra level of indentation here because this is substituted
# during software buildout to generate instance buildout, but the
# slapos.recipe.template recipe doing the substitution does string replacements
# without knowledge of the buildout syntax, so we want the second line to be
# indented in the final generated instance buildout.
extra-context =
raw slapos_repository_url ${slapos.cookbook-repository:repository}
raw slapos_repository_branch ${slapos.cookbook-repository:branch}
[exporter-default-configuration]
# Define shorter interaction to speed up tests
backup_wait_time = 1
software/helloworld/software.cfg
View file @ fc5a017f
......@@ -29,9 +29,6 @@ parts =
helloweb-go
[gowork]
golang = ${golang1.17:location}
# Macro for jinja templates. The filename is set in buildout.hash.cfg
# in the section using this template
[jinja-template]
......
software/kvm/README.rst
View file @ fc5a017f
......@@ -194,8 +194,12 @@ removing files:
from the partition (typically ``/srv/slapgrid/slappartNN/`` directory).
They will reappear automatically after some time, but as the old
``external-disk-amount`` approach is now disabled, they won't be updated.
The failure observed to confirm the situation can be found in
``.slappartNN_kvm-HASH.log`` with presence of message like::
qemu-system-x86_64: -drive file=/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2,if=virtio,cache=writeback: Failed to get "write" lock
Is another process using the image [/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2]?
ValueError: external-disk problems: conflicts with external-disk-number = XX, conflicts with already configured disks amount XX in /srv/slapgrid/slappartNN/etc/.data-disk-amount
Where ``XX`` is the previously used ``external-disk-number`` and ``NN`` is the partition.
software/kvm/buildout.hash.cfg
View file @ fc5a017f
......@@ -59,7 +59,7 @@ md5sum = 6328f99728284847b8dd1146aadeae1b
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = 4ce3fc8072e1e010ee99651cb01d3b3d
md5sum = 1663af08ea7afa8d3fa091bf0c2ea1ca
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
software/kvm/development.cfg
View file @ fc5a017f
# Development profile of slaprunner.
# Development profile of kvm.
# Exactly the same as software.cfg, but fetch the slapos.cookbook and
# slapos.toolbox from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
......
software/kvm/template/template-kvm-run.in
View file @ fc5a017f
......@@ -48,13 +48,25 @@ external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
external_disk = {{ parameter_dict['external-disk'] }}
if int(external_disk_number) > 0 and len(external_disk) > 0:
raise ValueError("external-disk-number and external-disk are mutually exclusive.")
etc_directory = '{{ parameter_dict.get("etc-directory") }}'.strip()
last_disk_num_f = os.path.join(etc_directory, '.data-disk-amount')
if os.path.exists(last_disk_num_f):
with open(last_disk_num_f, 'r') as lf:
last_amount = int(lf.readline())
else:
last_amount = 0
if len(external_disk) > 0:
conflict_list = []
if int(external_disk_number) > 0:
conflict_list.append('conflicts with external-disk-number = %s' % (external_disk_number,))
if last_amount > 0:
conflict_list.append('conflicts with already configured disks amount %s in %s' % (last_amount, last_disk_num_f))
if len(conflict_list) > 0:
raise ValueError('external-disk problems: ' + ', '.join(conflict_list))
instance_root = '{{ parameter_dict['instance-root'] }}'
disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = []
etc_directory = '{{ parameter_dict.get("etc-directory") }}'.strip()
httpd_port = {{ parameter_dict.get("httpd-port") }}
netcat_bin = '{{ parameter_dict.get("netcat-binary") }}'.strip()
cluster_doc_host = '{{ parameter_dict.get("cluster-doc-host") }}'
......
software/kvm/test/test.py
View file @ fc5a017f
......@@ -1875,38 +1875,6 @@ class TestParameterCluster(TestParameterDefault):
class ExternalDiskMixin(KvmMixin):
def getRunningDriveList(self, kvm_instance_partition):
_match_drive = re.compile('file.*if=virtio.*').match
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
drive_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_drive(entry)
if m:
path = m.group(0)
drive_list.append(
path.replace(kvm_instance_partition, '${partition}')
)
return drive_list
@skipUnlessKvm
class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
__partition_reference__ = 'ed'
kvm_instance_partition_reference = 'ed0'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def getInstanceParameterDict(cls):
return {
'external-disk-number': 2,
'external-disk-size': 1
}
@classmethod
def _prepareExternalStorageList(cls):
external_storage_path = os.path.join(cls.working_directory, 'STORAGE')
......@@ -1966,6 +1934,38 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
with open(cls.slap._slapos_config, 'w') as fh:
fh.write(''.join(slapos_config))
def getRunningDriveList(self, kvm_instance_partition):
_match_drive = re.compile('file.*if=virtio.*').match
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
drive_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_drive(entry)
if m:
path = m.group(0)
drive_list.append(
path.replace(kvm_instance_partition, '${partition}')
)
return drive_list
@skipUnlessKvm
class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
__partition_reference__ = 'ed'
kvm_instance_partition_reference = 'ed0'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def getInstanceParameterDict(cls):
return {
'external-disk-number': 2,
'external-disk-size': 1
}
@classmethod
def _setUpClass(cls):
super()._setUpClass()
......@@ -2015,19 +2015,29 @@ class TestExternalDiskJson(
pass
@skipUnlessKvm
class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
class ExternalDiskModernMixin(object):
__partition_reference__ = 'edm'
kvm_instance_partition_reference = 'edm0'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def setUpClass(cls):
super().setUpClass()
@classmethod
def _setUpClass(cls):
super()._setUpClass()
cls.working_directory = tempfile.mkdtemp()
# setup the external_storage_list, to mimic part of slapformat
cls._prepareExternalStorageList()
# re-run the instance, as information has been updated
cls.waitForInstance()
@classmethod
def tearDownClass(cls):
cls._dropExternalStorageList()
super().tearDownClass()
shutil.rmtree(cls.working_directory)
def getExternalDiskInstanceParameterDict(
self, first, second, third, update_dict=None):
parameter_dict = {
......@@ -2052,39 +2062,50 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
parameter_dict.update(update_dict)
return parameter_dict
def test(self):
def prepareEnv(self):
# Disks can't be created in /tmp, as it's specially mounted on testnodes
# and then KVM can't use them:
# -drive file=/tmp/tmpX/third_disk,if=virtio,cache=none: Could not open
# '/tmp/tmpX/third_disk': filesystem does not support O_DIRECT
self.working_directory = tempfile.mkdtemp(dir=self.slap.instance_directory)
self.addCleanup(shutil.rmtree, self.working_directory)
kvm_instance_partition = os.path.join(
self.kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
# find qemu_img from the tested SR via it's partition parameter, as
# otherwise qemu-kvm would be dependency of test suite
with open(
os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
qemu_img = [
self.qemu_img = [
q for q in fh.readlines()
if 'raw qemu_img_executable_location' in q][0].split()[-1]
self.first_disk = os.path.join(self.working_directory, 'first_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
second_disk = 'second_disk'
self.second_disk = os.path.join(kvm_instance_partition, second_disk)
self.qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
self.second_disk_name = 'second_disk'
self.second_disk = os.path.join(
self.kvm_instance_partition, self.second_disk_name)
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", os.path.join(
kvm_instance_partition, self.second_disk), "1M"])
self.qemu_img, "create", "-f", "qcow2", os.path.join(
self.kvm_instance_partition, self.second_disk), "1M"])
self.third_disk = os.path.join(self.working_directory, 'third_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
self.qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
self.rerequestInstance({'_': json.dumps(
self.getExternalDiskInstanceParameterDict(
self.first_disk, second_disk, self.third_disk))})
self.first_disk, self.second_disk_name, self.third_disk))})
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@skipUnlessKvm
class TestExternalDiskModern(
ExternalDiskModernMixin, InstanceTestCase, ExternalDiskMixin):
def test(self):
self.prepareEnv()
self.waitForInstance()
drive_list = self.getRunningDriveList(kvm_instance_partition)
drive_list = self.getRunningDriveList(self.kvm_instance_partition)
self.assertEqual(
drive_list,
[
......@@ -2097,16 +2118,39 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
self.working_directory)
]
)
update_dict = {
@skipUnlessKvm
class TestExternalDiskModernConflictAssurance(
ExternalDiskModernMixin, InstanceTestCase, ExternalDiskMixin):
def test(self):
self.prepareEnv()
# Create conflicting configuration
parameter_dict = {
"external-disk-number": 1,
"external-disk-size": 100,
"external-disk-size": 10,
"external-disk-format": "qcow2",
}
parameter_dict = self.getExternalDiskInstanceParameterDict(
self.first_disk, second_disk, self.third_disk, update_dict)
# assert mutual exclusivity
self.rerequestInstance({'_': json.dumps(parameter_dict)})
self.waitForInstance()
data_disk_ids = os.path.join(
self.kvm_instance_partition, 'etc', '.data-disk-ids')
data_disk_amount = os.path.join(
self.kvm_instance_partition, 'etc', '.data-disk-amount')
self.assertTrue(os.path.exists(data_disk_ids))
self.assertTrue(os.path.exists(data_disk_amount))
with open(data_disk_amount) as fh:
self.assertEqual(1, int(fh.read()))
parameter_dict.update(self.getExternalDiskInstanceParameterDict(
self.first_disk, self.second_disk_name, self.third_disk))
parameter_dict["external-disk-number"] = 0
# assert mutual exclusivity of old and modern
self.rerequestInstance({'_': json.dumps(parameter_dict)})
self.raising_waitForInstance(3)
# Fix the situation
with open(data_disk_amount, 'w') as fh:
fh.write("0")
self.waitForInstance()
@skipUnlessKvm
......
software/seleniumserver/buildout.hash.cfg
View file @ fc5a017f
......@@ -15,8 +15,8 @@
[template]
filename = instance.cfg.in
md5sum = 9b41238f67ea8e12f8ea9590fd552b33
md5sum = 10e19df182c692b71ea552da183a0bcf
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = 9b4742e8a249aef38c7c8c12d74b0605
md5sum = 2d84d5fb9306b88ef71e101acb46f684
software/seleniumserver/instance-selenium.cfg.in
View file @ fc5a017f
......@@ -145,44 +145,74 @@ username = selenium
bytes = 12
[selenium-server-frontend-config]
recipe = slapos.recipe.template
output = $${directory:etc}/$${:_buildout_section_name_}
inline =
https://[$${:ip}]:$${:port} {
bind $${:ip}
tls self_signed # TODO
proxy / $${selenium-server-hub-instance:base-url} {
transparent
}
basicauth $${selenium-server-admin-password:username} $${selenium-server-admin-password:passwd} {
realm "Grid Admin"
$${:path-admin}
}
basicauth $${selenium-server-selenium-password:username} $${selenium-server-selenium-password:passwd} {
realm "Selenium Server"
$${:path-hub}
}
}
[selenium-server-frontnend-certificate]
recipe = plone.recipe.command
command =
if [ ! -e $${:cert-file} ]
then
${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:2048 -keyout $${:cert-file} \
-out $${:cert-file}
fi
update-command = $${:command}
cert-file = $${directory:var}/$${:_buildout_section_name_}.pem
common-name = $${selenium-server-frontend-configuration:ip}
location =
$${:cert-file}
[selenium-server-frontend-configuration]
ip = $${slap-configuration:ipv6-random}
hostname = [$${:ip}]
port = 9443
path-admin = /grid/console
path-hub = /wd/hub
pidfile = $${directory:run}/haproxy.pid
[selenium-server-frontend-config]
recipe = slapos.recipe.template:jinja2
url = ${stack-haproxy-default-backend-config:target}
output = $${directory:etc}/$${:_buildout_section_name_}
context =
key pidfile selenium-server-frontend-configuration:pidfile
key content :content
content =
frontend app
log global
bind $${selenium-server-frontend-configuration:ip}:$${selenium-server-frontend-configuration:port} ssl crt $${selenium-server-frontnend-certificate:cert-file} alpn h2,http/1.1
use_backend hub if { path_beg $${selenium-server-frontend-configuration:path-hub} }
use_backend admin if { path_beg $${selenium-server-frontend-configuration:path-admin} }
userlist hub
user $${selenium-server-selenium-password:username} insecure-password $${selenium-server-selenium-password:passwd}
backend hub
acl auth_ok http_auth(hub)
http-request auth realm "Selenium Server" unless auth_ok
server hub $${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port}
userlist admin
user $${selenium-server-admin-password:username} insecure-password $${selenium-server-admin-password:passwd}
backend admin
acl auth_ok http_auth(admin)
http-request auth realm "Grid Admin" unless auth_ok
server admin $${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port}
[selenium-server-frontend-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line =
${caddy:output} -conf $${selenium-server-frontend-config:output}
${haproxy:location}/sbin/haproxy -f $${selenium-server-frontend-config:output}
hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
hash-files =
$${selenium-server-frontend-config:output}
ip = $${selenium-server-frontend-config:ip}
hostname = $${selenium-server-frontend-config:hostname}
port = $${selenium-server-frontend-config:port}
ip = $${selenium-server-frontend-configuration:ip}
hostname = $${selenium-server-frontend-configuration:hostname}
port = $${selenium-server-frontend-configuration:port}
admin-url = https://$${selenium-server-admin-password:username}:$${selenium-server-admin-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-admin}
url = https://$${selenium-server-selenium-password:username}:$${selenium-server-selenium-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-config:path-hub}
admin-url = https://$${selenium-server-admin-password:username}:$${selenium-server-admin-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-configuration:path-admin}
url = https://$${selenium-server-selenium-password:username}:$${selenium-server-selenium-password:passwd}@$${:hostname}:$${:port}$${selenium-server-frontend-configuration:path-hub}
[userinfo]
......
software/seleniumserver/instance.cfg.in
View file @ fc5a017f
......@@ -24,3 +24,5 @@ recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/seleniumserver/software.cfg
View file @ fc5a017f
......@@ -10,11 +10,11 @@ extends =
../../component/chromedriver/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/java/buildout.cfg
../../component/caddy/buildout.cfg
../../component/openssh/buildout.cfg
../../component/fonts/buildout.cfg
../../component/fontconfig/buildout.cfg
../../stack/slapos.cfg
../../stack/haproxy/default-backend.cfg
./buildout.hash.cfg
../../stack/monitor/buildout.cfg
......
software/slapos-sr-testing/README.md
View file @ fc5a017f
......@@ -12,7 +12,7 @@ The results of this test suite running on Nexedi ERP5 are published as `SlapOS.S
## Running test locally
Here's an example session of how a developer could use this software release in
slaprunner to develop a slapos profile, in the example `helloworld`, make
Theia to develop a slapos profile, in the example `helloworld`, make
changes to the code, run tests and publish changes.
```bash
......
software/slapos-sr-testing/software.cfg
View file @ fc5a017f
......@@ -117,11 +117,6 @@ setup = ${slapos-repository:location}/software/restic-rest-server/test/
egg = slapos.test.seleniumserver
setup = ${slapos-repository:location}/software/seleniumserver/test/
[slapos.test.slaprunner-setup]
<= setup-develop-egg
egg = slapos.test.slaprunner
setup = ${slapos-repository:location}/software/slaprunner/test/
[slapos.test.metabase-setup]
<= setup-develop-egg
egg = slapos.test.metabase
......@@ -337,7 +332,6 @@ eggs +=
${slapos.test.restic_rest_server-setup:egg}
${slapos.test.seleniumserver-setup:egg}
${slapos.test.slapos-master-setup:egg}
${slapos.test.slaprunner-setup:egg}
${slapos.test.theia-setup:egg}
${slapos.test.turnserver-setup:egg}
${slapos.test.upgrade_erp5-setup:egg}
......@@ -428,7 +422,6 @@ tests =
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
......
software/slapos-testing/README.md
View file @ fc5a017f
......@@ -10,7 +10,7 @@ The results of this test suite running on Nexedi ERP5 are published as
Here's an example session of how a developer could use this software release in
slaprunner to develop a slapos egg, in the example `slapos.core`, to make
Theia to develop a slapos egg, in the example `slapos.core`, to make
changes to the code, run tests and publish changes.
```bash
......
software/slaprunner/README.rst deleted 100644 → 0
View file @ d2c3e801
slaprunner
==========
Introduction
------------
This software release is used to deploy Slaprunner instances.
Slaprunner is an all-in-one IDE used to develop and test profiles and recipes for SlapOS.
This software is only running in Python2 and has been deprecated in favor of theia.
See https://lab.nexedi.com/nexedi/slapos/tree/master/software/theia
software/slaprunner/TODO.rst deleted 100644 → 0
View file @ d2c3e801
- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
- add test for parameter auto-deploy-instance
- Add download facility in file browser
software/slaprunner/buildout.hash.cfg deleted 100644 → 0
View file @ d2c3e801
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 473325442cba8ee32642d1e5e3fa94f4
[template-runner]
filename = instance-runner.cfg
md5sum = 03d3774fcb01a8ec68a23de31fb346b8
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
[instance-runner-import]
filename = instance-runner-import.cfg.in
md5sum = f4d71d97cf16ef379b41c4c22352727c
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 5f57249108e61976d97465624cc5a2a1
[template-resilient]
filename = instance-resilient.cfg.jinja2
md5sum = bd0ad0b80d2b39189f9665c48f1b3830
[template_nginx_conf]
filename = nginx_conf.in
md5sum = a042e52e0594ec0486cff7dc896cb6ce
[template_httpd_conf]
filename = httpd_conf.in
md5sum = 35f5f5c6e183af6064398c88fa85de59
[template_launcher]
filename = launcher.in
md5sum = 7a2f5e25a818cb29964666ada7852a5c
[template-slapos-cfg]
filename = template/slapos.cfg.in
md5sum = 03322b8a0f42ea579d35981211359190
[template-slapformat-definition.cfg]
filename = template/slapformat-definition.cfg.in
md5sum = bbb767dee9730f5a816ce4341403d89b
[template-parameters]
filename = parameters.xml.in
md5sum = 0d7b2432aa809b859509ca1114accd8f
[template-bash-profile]
filename = template/bash_profile.in
md5sum = d0ff3c86c76168d17d27b47b93a9b7aa
[template-supervisord]
filename = template/supervisord.conf.in
md5sum = 5e6c84098440c6bc163898dcafca8c9b
[template-listener-slapgrid]
filename = template/listener_slapgrid.py.in
md5sum = 49d50410cf7467175a841eb7cd0d93d4
[monitor-check-webrunner-internal-instance]
filename = template/monitor-check-webrunner-internal-instances.py
md5sum = acaac32cf1bd45714272468a89f4f119
[template-resilient-software-release-information]
filename = template/resilient_software_release_information.py.in
md5sum = 869a3afbf3c2fff6c72602662c6a8730
[template-slapuser-script]
filename = template/slapos-slapuser-script.in
md5sum = 75aab99c995ca841f93fc77fc9116c37
[template-buildout-shared-part-list]
filename = template/buildout-shared-part-list.in
md5sum = f619c8c5897c4851442b7090c8509758
software/slaprunner/development.cfg deleted 100644 → 0
View file @ d2c3e801
# Development profile of slaprunner.
# Exactly the same as software.cfg, but fetch the slapos.cookbook and
# slapos.toolbox from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
[buildout]
extends = software.cfg
../../stack/slapos-dev.cfg
parts +=
# Development eggs
${:common-parts}
[versions]
slapos.cookbook =
slapos.core =
slapos.toolbox =
software/slaprunner/httpd_conf.in deleted 100644 → 0
View file @ d2c3e801
PidFile "{{ parameters.path_pid }}"
ServerName example.com
ServerAdmin someone@email
<IfDefine !HTTPDPort>
Listen [{{ parameters.global_ip }}]:{{ parameters.global_port }}
Define HTTPDPort
</IfDefine>
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule mime_module modules/mod_mime.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule env_module modules/mod_env.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule cache_module modules/mod_cache.so
LoadModule file_cache_module modules/mod_file_cache.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule dir_module modules/mod_dir.so
LoadModule autoindex_module modules/mod_autoindex.so
ErrorLog "{{ parameters.path_error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b" common
CustomLog "{{ parameters.path_access_log }}" common
# SSL Configuration
Define SSLConfigured
SSLCertificateFile {{ parameters.cert_file }}
SSLCertificateKeyFile {{ parameters.key_file }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLEngine On
Include {{ parameters.httpd_cors_file }}
DocumentRoot {{ parameters.runner_home }}/public
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Require all denied
</Directory>
Alias /public {{ parameters.runner_home }}/public
<Directory {{ parameters.runner_home }}/public>
<Limit GET HEAD OPTIONS REPORT PROPFIND>
Order Allow,Deny
Allow from all
AllowOverride All
Satisfy Any
Options Indexes FollowSymLinks
DirectoryIndex index.html
</Limit>
<Files .htaccess>
order allow,deny
deny from all
</Files>
</Directory>
DavLockDB {{ parameters.dav_lock }}
Alias /share {{ parameters.runner_home }}
<Directory {{ parameters.runner_home }}>
DirectoryIndex disabled
DAV On
Options Indexes FollowSymLinks
AuthType Basic
AuthName "Webrunner Dav"
AuthUserFile "{{ parameters.htpasswd_file }}"
# Prevent using the web browser cache if requesting the same document
# from different domains or with different users
Header set Cache-Control "private, max-age=0, must-revalidate"
Header set Vary "Origin,Cookie,Authorization"
<LimitExcept OPTIONS>
Require valid-user
</LimitExcept>
</Directory>
software/slaprunner/instance-resilient.cfg.jinja2 deleted 100644 → 0
View file @ d2c3e801
# vim: set ft=cfg:
# Default parameters
# NOTE: in case of resilient instance, auto-deploy-instance
# set to false (else it could be dangerous)
{% do slapparameter_dict.__setitem__('auto-deploy', slapparameter_dict.get('auto-deploy', 'true')) %}
{% do slapparameter_dict.__setitem__('auto-deploy-instance', slapparameter_dict.get('auto-deploy-instance', 'false')) %}
{% do slapparameter_dict.__setitem__('instance-type', 'resilient') %}
# let decide the number of clones we want. We can't have more than 2 clones
{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
{% if number_of_instances > 2 %}
{% set number_of_instances = 2 %}
{% endif %}
{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url'] -%}
{% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
{% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
{% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
{% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% import 'parts' as parts %}
{% import 'replicated' as replicated with context %}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
extends =
{{ monitor_template }}
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
publish-early
{{ parts.replicate("runner", number_of_instances + 1) }}
publish-connection-information
[monitor-htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.monitor_user
bytes = 8
username = admin
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
init-password monitor-htpasswd:passwd
{% do monitor_parameter.__setitem__('monitor-username', slapparameter_dict.get('monitor-username', 'admin'))%}
{% do monitor_parameter.__setitem__('monitor-password', slapparameter_dict.get('monitor-password', '${publish-early:init-password}'))%}
{{ replicated.replicate("runner", number_of_instances + 1, "runner-export", "runner-import", slapparameter_dict=slapparameter_dict, monitor_parameter_dict=monitor_dict) }}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
# Bubble up the parameters
[request-runner]
{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
{% do slaprunner_return.append('custom-frontend-url') -%}
{% endif -%}
return = {{ slaprunner_return | join(' ')}} {{ monitor_return | join(' ') }}
[publish-connection-information]
recipe = slapos.cookbook:publish
backend-url = ${request-runner:connection-backend-url}
url = ${request-runner:connection-url}
init-user = ${request-runner:connection-init-user}
init-password = ${publish-early:init-password}
ssh-command = ${request-runner:connection-ssh-command}
webdav-url = ${request-runner:connection-webdav-url}
public-url = ${request-runner:connection-public-url}
{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
custom-frontend-url = ${request-runner:connection-custom-frontend-url}
{% endif %}
monitor-base-url = ${request-runner:connection-monitor-base-url}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${request-runner:connection-monitor-url}&username=${request-runner:connection-monitor-user}&password=${request-runner:connection-monitor-password}
[slap-parameter]
# Default parameters for distributed deployment
# I.e state "backup1 of maria should go there, ..."
# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
{% for nb in range(1,number_of_instances+1) %}
runner{{nb}}-computer-guid =
pbs-runner{{nb}}-computer-guid =
{% endfor %}
# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
domain =
authorized-key =
instance-amount = 10
debug = false
resilient-clone-number = 1
software/slaprunner/instance-runner-export.cfg.in deleted 100644 → 0
View file @ d2c3e801
[buildout]
extends = {{ template_runner_path }}
{{ pbsready_export_template_path }}
# parts from {{ template_runner_path }} are overriden by
# {{ pbsready_export_template_path }} so we reinclude them here
parts += ${:common-runner-parts}
[directory]
recipe = slapos.cookbook:mkdirectory
# XXX - keep srv path with slash at the end.
srv = ${:home}/srv/
[proxy-free-port]
recipe = slapos.cookbook:free_port
minimum = 49980
maximum = 49989
ip = ${slap-network-information:local-ipv4}
[runner-free-port]
recipe = slapos.cookbook:free_port
minimum = 50005
maximum = 50014
ip = ${slap-network-information:global-ipv6}
[slaprunner]
proxy_port = ${proxy-free-port:port}
runner_port = ${runner-free-port:port}
[supervisord-free-port]
recipe = slapos.cookbook:free_port
minimum = 39986
maximum = 39995
ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[exporter-raw]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter-raw
command-line = {{ software_release_bin }}/${slap-parameter:namebase}-exporter --srv-path ${directory:srv} --etc-path ${directory:etc} --backup-path ${directory:backup} --backup-wait-time {{ backup_wait_time }}
environment =
PATH={{ rsync_bin_folder }}:/bin:/usr/bin
[exporter]
recipe = slapos.cookbook:wrapper
command-line = {{ bash_executable_location }} -c "exec ${exporter-raw:wrapper-path} > >(tee -ai "${directory:log}/resilient.log") 2>&1"
wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter
# wrapper parameter is needed by resilience stack
wrapper = ${:wrapper-path}
[monitor-httpd-free-port]
recipe = slapos.cookbook:free_port
minimum = 8437
maximum = 8446
ip = ${slap-network-information:global-ipv6}
# XXX Redefine Monitor parameters for runner-export
[monitor-instance-parameter]
monitor-httpd-port = ${monitor-httpd-free-port:port}
{% if slapparameter_dict.get('name', '') -%}
monitor-title = {{ slapparameter_dict['name'] }}
{% endif -%}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-url-list', '') -%}
monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
{% endif -%}
instance-configuration =
httpdcors cors-domain ${slaprunner-httpd-cors:location} ${httpd-graceful-wrapper:output}
configuration-file-path = ${buildout:directory}/knowledge0.cfg
[monitor-conf-parameters]
private-path-list +=
$${logrotate-directory:logrotate-backup}
# Extends publish section with resilient parameters
[publish-connection-information]
<= resilient-publish-connection-parameter
[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
url = {{ monitor_check_resilient_feed_template_path }}
output = ${monitor-directory:bin}/check-create-resilient-feed-files
context =
key input_feed_directory directory:notifier-feeds
key monitor_feed_directory monitor-directory:public
raw base_url http://[${notifier:host}]:${notifier:port}/get/
raw python_executable ${buildout:executable}
software/slaprunner/instance-runner-import.cfg.in deleted 100644 → 0
View file @ d2c3e801
[buildout]
extends = {{ template_runner_path }}
{{ pbsready_import_template_path }}
parts +=
nginx_conf
nginx-launcher
certificate-authority
ca-nginx
certificate-authority-service
ca-nginx-service
gunicorn-launcher
gunicorn-graceful
slaprunner-promise
slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key
runner-sshd-promise
runner-sshd-service
runtestsuite
shellinabox
shellinabox-service
slapos-cfg
cron-entry-prepare-software
deploy-instance-parameters
instance-software-type
bash-profile
supervisord-wrapper
importer-consistency-promise
software-release-deployment-promise
template-slapuser-script
resilient-software-release-information
monitor-base
[directory]
recipe = slapos.cookbook:mkdirectory
# XXX - keep srv path with slash at the end.
srv = ${:home}/srv/
# For the needs of importer, we run the full slaprunner
# In case both exporter and importer (aka main instance and clone instance)
# run with the same IP (usually for testing purposes),
# run slaprunner using different ports.
[proxy-free-port]
recipe = slapos.cookbook:free_port
minimum = 49990
maximum = 49999
ip = ${slap-network-information:local-ipv4}
[runner-free-port]
recipe = slapos.cookbook:free_port
minimum = 50015
maximum = 50024
ip = ${slap-network-information:global-ipv6}
[slaprunner]
proxy_port = ${proxy-free-port:port}
runner_port = ${runner-free-port:port}
[supervisord-free-port]
recipe = slapos.cookbook:free_port
minimum = 39996
maximum = 40005
ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[runner-sshd-port]
minimum = 22232
maximum = 22241
# Deactivate the call to prepareSoftware, and let the importer script
# handle the build&run of the instance.
[cron-entry-prepare-software]
recipe =
[importer]
recipe = slapos.recipe.template:jinja2
url = {{ importer_script_path }}
output = ${directory:bin}/${slap-parameter:namebase}-importer
# backward compatibility for resilient stack
wrapper = ${:output}
restore-exit-code-file = ${directory:srv}/${:restore-exit-code-file-basename}
restore-exit-code-file-basename = importer-exit-code-file
restore-error-message-file = ${directory:srv}/${:restore-error-message-file-basename}
restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log
context =
import sys sys
import easy_install zc.buildout.easy_install
key backend_url slaprunner:access-url
key ipv4 slaprunner:ipv4
key ipv6 slaprunner:ipv6
key proxy_port slaprunner:proxy_port
key instance_folder slaprunner:instance_root
section directory directory
section supervisord supervisord
raw output_log_file ${directory:log}/${:resilient-log-basename}
raw shell_binary {{ bash_executable_location }}
raw sqlite3_binary {{ sqlite3_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
raw restore_exit_code_file ${:restore-exit-code-file}
raw restore_error_message_file ${:restore-error-message-file}
[importer-consistency-promise-bin]
# Test that the importer script and "after-import" subscripts
# are not older than 2 days (1 day + some slack), and have succeeded
recipe = collective.recipe.template
input = inline: #!/bin/sh
EXIT_CODE_FILE="${importer:restore-exit-code-file}"
RECENT_EXIT_CODE_FILE=$(find ${directory:srv} -maxdepth 1 -name "${importer:restore-exit-code-file-basename}" -mtime -2)
RESILIENT_LOG_URL=${monitor-publish-parameters:monitor-base-url}/share/private/log/${importer:resilient-log-basename}
if [ ! -f "$EXIT_CODE_FILE" ]; then
exit 0;
else
if [ -z "$RECENT_EXIT_CODE_FILE" ]; then
echo "Consistency check is too old.";
exit 1;
else
if [ -f "${importer:restore-error-message-file}" ]; then
cat ${importer:restore-error-message-file}
fi
echo "More information can be found here : $RESILIENT_LOG_URL";
exit $(cat $EXIT_CODE_FILE);
fi
fi
exit 1; # Something else went wrong
output = ${directory:bin}/importer-consistency-promise
mode = 755
[importer-consistency-promise]
<= monitor-promise-base
promise = check_command_execute
name = importer-consistency-promise.py
config-command = ${importer-consistency-promise-bin:output}
[software-release-deployment-bin]
recipe = collective.recipe.template
input = inline: #!/bin/sh
PROJECT_FILE=$(find "${directory:etc}" -maxdepth 1 -name .project)
if [ -z "$PROJECT_FILE" ]; then
exit 0;
else
INSTALLATION_FILE=$(find "${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
if [ -n "$INSTALLATION_FILE" ]; then
exit 0;
else
exit 1;
fi
fi
exit 1
output = ${directory:bin}/software-release-deployment-promise
mode = 755
[software-release-deployment-promise]
<= monitor-promise-base
promise = check_command_execute
name = software-release-deployment-promise.py
config-command =${software-release-deployment-bin:output}
[resilient-software-release-information]
recipe = slapos.recipe.template
url = {{ software_release_information_template }}
output = ${directory:cgi-bin}/resilient_software_release_information.py
resilient-log-url = ${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
[slap-parameter]
auto-deploy-instance = false
auto-deploy = true
[resilient-publish-connection-parameter]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = 8360
monitor-title = {{ slapparameter_dict.get('name', 'Webrunner import') }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
instance-configuration =
raw takeover-url http://[${resilient-web-takeover-httpd-configuration-file:listening-ip}]:${resilient-web-takeover-httpd-configuration-file:listening-port}/
raw takeover-password ${resilient-web-takeover-password:passwd}
configuration-file-path = ${buildout:directory}/knowledge0.cfg
[monitor-conf-parameters]
private-path-list +=
$${logrotate-directory:logrotate-backup}
[post-notification-run]
recipe = slapos.cookbook:wrapper
command-line = {{ software_release_bin }}/runner-importer-post-notification-run --diff-file ${:diff-file} --proof-signature-file ${:proof-signature-file} --srv-path ${directory:srv} --backup-path ${directory:backup}
wrapper-path = ${rootdirectory:bin}/post-notification-run
output = ${:wrapper-path}
mode = 0700
software/slaprunner/instance-runner-input-schema.json deleted 100644 → 0
View file @ d2c3e801
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"instance-name": {
"title": "Instance Name",
"description": "Name of the instance, to show in the window title",
"type": "string"
},
"custom-frontend-backend-url": {
"title": "Custom Frontend Backend URL",
"description": "return an ipv4 frontend of the given ipv6(+optional port)",
"type": "string",
"format": "uri"
},
"custom-frontend-backend-type": {
"title": "Custom Frontend Backend Type",
"description": "The type of the frontend slave instance to ask",
"type": "string",
"enum": [
"zope"
]
},
"user-authorized-key": {
"title": "User Authorized Key",
"description": "SSH public key in order to connect to the SSH server of this runner instance.",
"textarea": true,
"type": "string"
},
"instance-amount": {
"title": "Partition Amount",
"description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.",
"type": "integer",
"minimum": 1,
"maximum": 40
},
"auto-deploy": {
"title": "Automatically Deploy Software",
"description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)",
"type": "boolean"
},
"auto-deploy-instance": {
"title": "Automatically Deploy Instances",
"description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)",
"type": "boolean"
},
"autorun": {
"title": "Automatically Run Sofware/Instance",
"description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)",
"type": "boolean"
},
"slapos-software": {
"title": "Pre-selected Software Release",
"description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
"type": "string"
},
"slapos-repository": {
"title": "SlapOS Git Repository URL",
"description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is https://lab.nexedi.com/nexedi/slapos.git)",
"type": "string",
"format": "uri",
"pattern": "^(http|https|ftp)://"
},
"slapos-reference": {
"title": "SlapOS Git Branch Name",
"description": "Branch or hash on which the default repository will checkout (default is master)",
"type": "string"
},
"slapos-software-type": {
"title": "Deployed Instance Software Type",
"description": "Software type of your instance inside the runner",
"type": "string"
},
"cpu-usage-ratio": {
"title": "CPU Usage Ratio",
"description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
"type": "integer",
"default": 4
},
"no-ipv4-frontend": {
"title": "No IPv4 frontend",
"description": "Prevent the slaprunner to order an IPv4 frontend for itself",
"enum": [
"true",
"false"
],
"default": "false"
},
"custom-frontend-basic-auth": {
"title": "Custom Frontend Basic Auth",
"description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
"type": "boolean"
},
"custom-frontend-instance-guid": {
"title": "Custom Frontend Instance GUID",
"description": "Instance guid of the frontend you whish to use",
"type": "string"
},
"custom-frontend-software-type": {
"title": "Custom Frontend Software Type",
"description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)",
"type": "string",
"enum": [
"RootSoftwareInstance",
"default"
]
},
"custom-frontend-software-url": {
"title": "Custom Frontend Software URL",
"description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)",
"type": "string",
"format": "uri"
},
"check-custom-frontend-promise": {
"title": "Check Custom Frontend Promise",
"description": "Enable a promise to check that HTTP frontend created from custom-frontend-backend-url is available",
"type": "string",
"enum": [
"true",
"false"
],
"default": "false"
},
"slaprunner-httpd-port": {
"title": "Webrunner Server Port",
"description": "Port of the Apache server serving the Webrunner interface.",
"type": "integer",
"default": 9686
},
"monitor-httpd-port": {
"title": "Monitor Port",
"description": "Port of the Apache server serving the monitoring interface.",
"type": "integer",
"default": 8386
},
"monitor-interface-url": {
"title": "Monitor Web Interface URL",
"description": "Give Url of HTML web interface that will be used to render this monitor instance.",
"type": "string",
"format": "uri",
"default": "https://monitor.app.officejs.com"
},
"monitor-cors-domains": {
"title": "Monitor CORS domains",
"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
"type": "string",
"default": "monitor.app.officejs.com"
}
}
}
software/slaprunner/instance-runner-output-schema.json deleted 100644 → 0
View file @ d2c3e801
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Runner instanciation",
"properties": {
"backend_url": {
"description": "ipv6 url to directly access your runner",
"type": "string"
},
"access_url": {
"description": "Url to set up your account and then login into your webrunner",
"type": "string"
},
"url": {
"description": "Url to access your runner once you set your account",
"type": "string"
},
"ssh_command": {
"description": "SSH command used to access your runner in ssh when you provided a ssh public key",
"type": "string"
},
"monitor_url": {
"description": "Url to access the monitoring interface of your instance. Use this url to get the recovery code needed to set up your runner",
"type": "string"
},
"webdav_url": {
"description": "Url for webdav access of your runner it access the root of your runner. Same users as the one for the runner",
"type": "string"
},
"public_url": {
"description": "Url of the public folder of your runner",
"type": "string"
}
},
"type": "object"
}
software/slaprunner/instance-runner-resilient-input-schema.json deleted 100644 → 0
View file @ d2c3e801
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"allOf": [
{
"$ref": "instance-runner-input-schema.json#/"
},
{
"properties": {
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 2,
"optional": true
},
"-sla-runner0-computer_guid": {
"title": "Target computer for main instance",
"description": "Target computer GUID for main instance.",
"type": "string",
"optional": true
},
"-sla-runner1-computer_guid": {
"title": "Target computer for first clone",
"description": "Target computer for first clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs1-computer_guid": {
"title": "Target computer for first PBS",
"description": "Target computer for first PBS instance.",
"type": "string",
"optional": true
},
"-sla-runner2-computer_guid": {
"title": "Target computer for second clone",
"description": "Target computer for second clone instance.",
"type": "string",
"optional": true
},
"-sla-pbs2-computer_guid": {
"title": "Target computer for second PBS",
"description": "Target computer for second PBS instance.",
"type": "string",
"optional": true
},
"resiliency-backup-periodicity": {
"title": "Periodicity of backup",
"description": "Periodicity of backup, in cron format.",
"type": "string",
"optional": true
},
"remove-backup-older-than": {
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string",
"default": "2W",
"optional": true
}
}
}
]
}
software/slaprunner/instance-runner.cfg deleted 100644 → 0
View file @ d2c3e801
This diff is collapsed.
software/slaprunner/instance.cfg deleted 100644 → 0
View file @ d2c3e801
[buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
extends =
${template-resilient-templates:output}
[switch_softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = $${:runner}
resilient = instance-resilient:output
runner = instance-base-runner:output
runner-import = template-runner-import:output
runner-export = template-runner-export:output
frozen = instance-frozen:output
pull-backup = template-pull-backup:output
# BBB
RootSoftwareInstance = $${:default}
[instance-base-runner]
recipe = slapos.recipe.template:jinja2
url = ${template-runner:output}
output = $${buildout:directory}/template-runner.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond
[instance-resilient]
recipe = slapos.recipe.template:jinja2
url = ${template-resilient:target}
output = $${buildout:directory}/instance-resilient.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw monitor_template ${monitor-template:output}
template-parts-destination = ${template-parts:target}
template-replicated-destination = ${template-replicated:target}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
[template-runner-export]
recipe = slapos.recipe.template:jinja2
url = ${instance-runner-export:target}
output = $${buildout:directory}/instance-runner-export.cfg
context =
key pbsready_export_template_path template-pbsready-export:output
key template_runner_path instance-base-runner:output
key slapparameter_dict slap-configuration:configuration
raw software_release_bin ${buildout:bin-directory}
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:target}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw rsync_bin_folder ${rsync:location}/bin
[template-runner-import]
recipe = slapos.recipe.template:jinja2
url = ${instance-runner-import:target}
output = $${buildout:directory}/instance-runner-import.cfg
context =
key pbsready_import_template_path template-pbsready-import:output
key template_runner_path instance-base-runner:output
key slapparameter_dict slap-configuration:configuration
raw software_release_bin ${buildout:bin-directory}
raw importer_script_path ${template-runner-import-script:target}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw sqlite3_executable_location ${sqlite3:location}/bin/sqlite3
raw rsync_executable_location ${rsync:location}/bin/rsync
raw software_release_information_template ${template-resilient-software-release-information:target}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/slaprunner/launcher.in deleted 100644 → 0
View file @ d2c3e801
#! {{ param_nginx_frontend['path_shell'] }}
# BEWARE: This file is operated by slapos node
# BEWARE: It will be overwritten automatically
# Run nginx
exec {{ param_nginx_frontend['bin_nginx'] }} \
-p {{ param_nginx_frontend['nginx_prefix'] }} \
-c {{ param_nginx_frontend['path_nginx_conf'] }}
software/slaprunner/nginx_conf.in deleted 100644 → 0
View file @ d2c3e801
worker_processes {{ param_nginx_frontend['nb_workers'] }};
pid {{ param_nginx_frontend['path_pid'] }};
error_log {{ param_nginx_frontend['path_error_log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ param_nginx_frontend['path_access_log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ param_nginx_frontend['global-ip'] }}]:{{ param_nginx_frontend['global-port'] }} ssl;
server_name _;
ssl_certificate {{ param_nginx_frontend['ssl-certificate'] }};
ssl_certificate_key {{ param_nginx_frontend['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
location / {
auth_basic "Restricted";
auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Host $http_host;
proxy_connect_timeout 200;
proxy_send_timeout 200;
proxy_read_timeout 200;
send_timeout 200;
proxy_pass http://unix:{{ socket }};
}
location ~ ^(/login|/doLogin|/static|/slapgridResult|/isSRReady) {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Host $http_host;
proxy_pass http://unix:{{ socket }};
}
location /shellinabox {
proxy_pass http://unix:{{ shellinabox_socket }}:/;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
auth_basic "Restricted";
auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
proxy_redirect off;
proxy_buffering off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
}
}
}
software/slaprunner/parameters.xml.in deleted 100644 → 0
View file @ d2c3e801
{% set inst_parameter_dict = {} -%}
{% if slapparameter_dict is defined -%}
{% for key in slapparameter_dict.keys() -%}
{% if key.startswith('parameter-') -%}
{% do inst_parameter_dict.__setitem__(key[10:], slapparameter_dict.pop(key)) -%}
{% endif -%}
{% endfor -%}
{% endif -%}
<?xml version='1.0' encoding='utf-8'?>
<instance>
{% if slapparameter_dict is defined %}
{% for parameter_name in inst_parameter_dict.keys() %}
<parameter id="{{ parameter_name }}">{{ inst_parameter_dict[parameter_name] }}</parameter>
{% endfor %}
{% endif %}
</instance>
software/slaprunner/software.cfg deleted 100644 → 0
View file @ d2c3e801
[buildout]
extends =
buildout.hash.cfg
../../component/bash/buildout.cfg
../../component/fish-shell/buildout.cfg
../../component/tmux/buildout.cfg
../../component/busybox/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/git/buildout.cfg
../../component/tig/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/nano/buildout.cfg
../../component/nginx/buildout.cfg
../../component/openssh/buildout.cfg
../../component/mosh/buildout.cfg
../../component/rsync/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/screen/buildout.cfg
../../component/shellinabox/buildout.cfg
../../component/vim/buildout.cfg
../../component/zip/buildout.cfg
../../stack/slapos.cfg
../../stack/flask.cfg
../../stack/resilient/buildout.cfg
../../stack/logrotate/buildout.cfg
../../stack/monitor/buildout.cfg
# make sure shared-part-list is available, even for old versions
# of slapos who do not set that.
shared-part-list =
# stacks are listed from most generic to most specific,
# to avoid versioning issues
common-parts =
slapos-command
slapos-cookbook
template
instance-runner-import
instance-runner-export
template-slapos-cfg
template-slapformat-definition.cfg
template-slapuser-script
# XXX: we have to manually add this for resilience
pbs-recipe-egg
parts =
${:common-parts}
[python]
part = python2.7
[template-base]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
[template]
< = template-base
output = ${buildout:directory}/template.cfg
[template-runner]
< = template-base
output = ${buildout:directory}/template-runner.cfg.in
[template-runner-import-script]
< = download-base
[instance-runner-import]
< = download-base
[instance-runner-export]
< = download-base
[template-resilient]
< = download-base
[template_nginx_conf]
< = download-base
[template_httpd_conf]
< = download-base
[template_launcher]
< = download-base
[template-slapos-cfg]
< = download-base
[template-slapformat-definition.cfg]
< = download-base
[template-parameters]
< = download-base
[template-bash-profile]
< = download-base
[template-supervisord]
< = download-base
[template-listener-slapgrid]
< = download-base
[monitor-check-webrunner-internal-instance]
< = download-base
[template-resilient-software-release-information]
< = download-base
[template-slapuser-script]
< = download-base
[template-buildout-shared-part-list]
< = template-base
output = ${buildout:directory}/buildout-shared-part-list
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
erp5.util
lock-file
slapos.recipe.build
slapos.toolbox[flask_auth]
gunicorn
# for gunicorn[gthread]
futures
supervisor
# BBB: eggs used as recipe should be kept otherwise sections depending
# on it can't be uninstalled
collective.recipe.shelloutput
scripts =
gunicorn
supervisord
supervisorctl
[versions]
Flask-Auth = 0.85
futures = 3.0.5
gunicorn = 19.10.0
prettytable = 0.7.2
pycurl = 7.43.0
software/slaprunner/software.cfg.json deleted 100644 → 0
View file @ d2c3e801
{
"name": "Web Runner",
"description": "Web Runner",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standalone Runner",
"request": "instance-runner-input-schema.json",
"response": "instance-runner-output-schema.json",
"index": 0
},
"resilient": {
"title": "Resilient",
"description": "Resilient Runner",
"request": "instance-runner-resilient-input-schema.json",
"response": "instance-runner-output-schema.json",
"index": 1
}
}
}
software/slaprunner/template/bash_profile.in deleted 100644 → 0
View file @ d2c3e801
# Beware, this file is automatically processed by slapgrid
# Do not modify it, your changes will be lost
# If you want to load your custom bash configuration, please use a .bashrc file
cd {{ workdir }}
export HOME={{- home }}
export PATH={{- path }}
export SHELL={{- shell }}
{%- if instance_name %}
export PROMPT_COMMAND='echo -en "\033]0;{{-instance_name}}\a"'
{% endif %}
export PS1="$ "
export TERMINFO={{- terminfo }}
if [ -f "$HOME/.bashrc" ] ; then
source $HOME/.bashrc
fi
# If not running interactively, don't do anything
case $- in
*i*) ;;
*) return;;
esac
# enable color support
if [ -x /usr/bin/dircolors ]; then
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
alias ls='ls --color=auto'
fi
# some more ls aliases
alias ll='ls -l'
alias la='ls -A'
alias l='ls -CF'
echo "Welcome to SlapOS slaprunner shell"
echo
software/slaprunner/template/cors-domain.jinja deleted 100644 → 0
View file @ d2c3e801
#!{{ python_executable }}
import cgi
import cgitb
import ConfigParser
import os
import re
import subprocess
cgitb.enable(display=0, logdir="/tmp/cgi.log")
form = cgi.FieldStorage()
apache_file = "{{ apache_file }}"
config_file = "{{ config_cfg }}"
if not os.path.exists(config_file):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
parser = ConfigParser.ConfigParser()
parser.read(config_file)
if not parser.has_section("cors"):
parser.add_section("cors")
if not parser.has_option("cors", "cors-domain"):
parser.set("cors", "cors-domain", "")
if "cors-domain" in form:
parser.set("cors", "cors-domain", form["cors-domain"].value)
cors_domain_parameter = parser.get("cors", "cors-domain")
if cors_domain_parameter:
cors_domain_list = cors_domain_parameter.split()
cors_string = ""
for domain in cors_domain_list:
if cors_string:
cors_string += '|'
cors_string += re.escape(domain)
with open(apache_file, 'w') as file:
file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
subprocess.call('{{ httpd_graceful }}')
with open(config_file, 'w') as file:
parser.write(file)
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"static/style.css\">"
print "</head><body>"
print "<h1>Cors Domains :</h1>"
print "<p>Enter domain names separated by space to authorized CORS on your dav storage for these domains</p>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
print "<div class=\"pure-control-group\">"
print '<label for="cors-domain">CORS Domains</label>'
print '<input type="text" name="cors-domain" value="%s">' % cors_domain_parameter
print "</div>"
print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
pure-button-primary\">Save</button></div></form>"
print "</body></html>"
software/slaprunner/template/listener_slapgrid.py.in deleted 100644 → 0
View file @ d2c3e801
#!{{ python_executable }}
import datetime
import json
import sys
import xmlrpclib
from supervisor import childutils
def write_stdout(s):
sys.stdout.write(s)
sys.stdout.flush()
def write_stderr(s):
sys.stderr.write(s)
sys.stderr.flush()
def write_slapgrid_result(process):
server = xmlrpclib.Server("http://{{- supervisord['server'] -}}")
# Tuple of tuples containing 2 elements : process name and the path of its info file
json_files = dict([("slapgrid-sr", "{{- slaprunner['software_info_json'] -}}"),
("slapgrid-cp", "{{- slaprunner['instance_info_json'] -}}")])
info = server.supervisor.getProcessInfo(process)
result = dict()
result['last_build'] = datetime.datetime.fromtimestamp(info['stop']).strftime("%Y-%m-%d %H:%M:%S")
result['success'] = info['exitstatus']
open(json_files[process], 'w').write(json.dumps(result))
def main():
while 1:
headers, payload = writer.wait()
try:
processname = dict(x.split(':') for x in payload.split())['processname']
write_slapgrid_result(processname)
except:
pass
writer.ok()
if __name__ == '__main__':
writer = childutils.EventListenerProtocol()
main()
software/slaprunner/template/monitor-check-webrunner-internal-instances.py deleted 100644 → 0
View file @ d2c3e801
#!/usr/bin/python
import os
import subprocess
import sys
def runPromise(promise_path):
promise_relative_path = promise_path.replace(os.path.expanduser('~'), '~')
print 'Running promise %s...' % promise_relative_path
promise_process = subprocess.Popen(promise_path, stderr=subprocess.PIPE)
stdout, stderr = promise_process.communicate()
return_code = promise_process.returncode
if return_code == 0:
print 'Success.'
return True
else:
sys.stderr.write('Failure while running promise %s. %s\n' % (promise_relative_path, stderr))
def getPromisePathListFromPartitionPath(partition_path):
promise_directory_path = os.path.join(partition_path, 'etc/promise')
try:
promise_name_list = os.listdir(promise_directory_path)
return [os.path.join(promise_directory_path, promise_name) for promise_name in promise_name_list]
except OSError:
return []
def main():
# XXX hardcoded
partition_root_path = os.path.expanduser('~/srv/runner/instance')
success = True
for partition_name in os.listdir(partition_root_path):
partition_path = os.path.join(partition_root_path, partition_name)
for promise_path in getPromisePathListFromPartitionPath(partition_path):
result = runPromise(promise_path)
if not result:
success = False
if not success:
sys.exit(1)
if __name__ == '__main__':
main()
software/slaprunner/template/resilient_software_release_information.py.in deleted 100644 → 0
View file @ d2c3e801
# Provides information related to the Webrunner Software Release to the
# takeover interface of the Resilient stack
def main():
return {
'Read the log from the importer': '<a href="${:resilient-log-url}">${:resilient-log-url}</a>',
}
software/slaprunner/template/runner-import.sh.jinja2 deleted 100644 → 0
View file @ d2c3e801
#!{{ shell_binary }}
LC_ALL=C
export LC_ALL
umask 077
# Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion
set -eu
# Redirect output to log
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1
echo -e "\n\n$0 run at : $(date)"
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
ERROR_MESSAGE=""
fail_with_exit_code () {
echo 1 > $RESTORE_EXIT_CODE_FILE
echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
exit 1
}
trap fail_with_exit_code ERR
log_message () {
ERROR_MESSAGE=$1
echo -e $1
}
# Delete the error message file, to not keep it even after a successful build
rm "$RESTORE_ERROR_MESSAGE_FILE" || true
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
set +x
}
log_message "Restoring WebRunner content..."
(
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$backup_path"
if [ -d instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
try:
os.chdir(partition)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$path"
fi
test -d project && rsync project "$path"
test -d public && rsync public "$path"
test -f proxy.db && rsync proxy.db "$path"
)
log_message "Restoring WebRunner config (etc directory)..."
(
cd "$backup_directory"/etc/
rsync config.json "$etc_directory"
# Hidden files are related to the webrunner's internals
cp -r .??* "$etc_directory"
)
# Invoke arbitrary script to perform specific restoration
# procedure.
runner_import_restore=$srv_directory/runner-import-restore
if [ -x "$runner_import_restore" ]; then
log_message "Running $runner_import_restore..."
"$srv_directory/runner-import-restore"
fi
# If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume
# that there is no instance deployed on runner0
if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then
log_message "No Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
log_message "Updating slapproxy database..."
HOME='{{ directory["home"] }}'
# XXX Hardcoded
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export MAKEFLAGS=-j4
SLAPOS='{{ directory["bin"] }}'/slapos
SQLITE3="{{ sqlite3_binary }}"
DATABASE="$HOME/srv/runner/proxy.db"
db_query () {
# Try opening locked tables for 5 seconds to prevent "database is locked" error
"$SQLITE3" "$DATABASE" <<EOF
.timeout 5000
$@
EOF
}
# If slapproxy database is empty then no software release was opened
if [ ! -s "$DATABASE" ]; then
log_message "Slapproxy database empty, no Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
# Slap proxy table contain version number, find the table name dynamically.
# This is known to work with version 11 or 12 of tables, but it will probably
# work with earlier versions as well.
DB_PARTITION_TABLE=$(db_query ".table partition__")
DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__")
DB_SOFTWARE_TABLE=$(db_query ".table software__")
# Change slapproxy database to point instances to new software release
# XXX hardcoded
PARTITION=$(basename $HOME)
OLD_SOFTWARE_RELEASE=$(db_query "select software_release from $DB_PARTITION_TABLE where reference='slappart0';")
if [ "$OLD_SOFTWARE_RELEASE" == "" ];
then
echo "No instance configured"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
SOFTWARE_RELEASE=$({{ sys.executable }} - $OLD_SOFTWARE_RELEASE $PARTITION <<EOF
if 1:
import os, re, sys
# We want to replace the last occurence only
old_software_release, partition = sys.argv[1], sys.argv[2]
for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
start, end = match.start(), match.end()
print old_software_release[:start] + partition + old_software_release[end:]
EOF
)
db_query "update $DB_PARTITION_TABLE set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
db_query "update $DB_SOFTWARE_TABLE set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || db_query "delete from $DB_SOFTWARE_TABLE where url='$OLD_SOFTWARE_RELEASE';"
# Change slapproxy database to have all instances stopped
db_query "update $DB_PARTITION_TABLE set requested_state='stopped';"
# Change slapproxy database to get correct IPs
IPV4='{{ ipv4 }}'
IPV6='{{ ipv6 }}'
db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';"
db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';"
MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
log_message "Removing old supervisord service description files..."
# XXX: Path hardcoded in slapos.core
rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
SLAPOSCFG='{{ supervisord["slapos-cfg"] }}'
SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}'
SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}'
contain_software_release=0
SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';")
if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then
contain_software_release=1
fi
if [ $contain_software_release -eq 0 ]; then
log_message "No Software Release were deployed, so skip to continue..."
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
log_message "Building newest Software Release..."
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
(tail -n 200 "$SLAPGRIDSRLOG" && false)
contain_instance=0
for folder in $srv_directory/runner/instance/slappart*/; do
if [ -f "$folder/buildout.cfg" ]; then
contain_instance=1
fi
done
# If instance do not contains template.cfg it means the user contains no instance.
# so it is safer to assume that he is using slaprunner for develop buildout rather them slapos.
if [ $contain_instance -eq 0 ]; then
log_message "None Instance were deployed with this software release, so skip to continue..."
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
# Remove defined scripts to force buildout to recreate them to have updated paths
rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
log_message "Fixing Instances as needed after import..."
# XXX hardcoded
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
(tail -n 200 "$SLAPGRIDCPLOG" && false)
# Invoke defined scripts for each partition inside of slaprunner
log_message "Invoke custom import scripts defined by each instances..."
for partition in "$srv_directory"/runner/instance/slappart*/
do
script=$partition/srv/runner-import-restore
if [ -x "$script" ]; then
log_message "Running custom instance script : $script..."
"$script"
fi
done
# Change back slapproxy database to have all instances started
DB_PARTITION_TABLE=$(db_query ".table partition__")
log_message "Set instances as to start after takeover..."
db_query "update $DB_PARTITION_TABLE set requested_state='started';"
# Write exit code to an arbitrary file that will be checked by promise/monitor
log_message "Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
software/slaprunner/template/slapformat-definition.cfg.in deleted 100644 → 0
View file @ d2c3e801
[computer]
# Address example: 2001:67c:1254:27::1/ffff:ffff:ffff:ffff::/64
address = {{ slaprunner['ipv6'] }}/64
{% for partition_index in range(int(slaprunner['partition-amount'])) %}
[partition_{{ partition_index }}]
address = {{ slaprunner['ipv4'] }}/255.255.255.255 {{ slaprunner['ipv6'] }}/64
user = {{ partition_user }}
pathname = slappart{{ partition_index }}
network_interface =
{% endfor %}
software/slaprunner/template/slapos-slapuser-script.in deleted 100644 → 0
View file @ d2c3e801
#!/bin/sh
# run slapos command inside slaprunner with appropriate config
export SLAPOS_CONFIGURATION=${SLAPOS_CONFIGURATION-"{{ config_location }}"}
export SLAPOS_CLIENT_CONFIGURATION=${SLAPOS_CLIENT_CONFIGURATION-$SLAPOS_CONFIGURATION}
exec {{ slapos_python_file_location }} "$@"
software/slaprunner/template/slapos.cfg.in deleted 100644 → 0
View file @ d2c3e801
[slapos]
software_root = {{ slaprunner['software_root'] }}
instance_root = {{ slaprunner['instance_root'] }}
shared_part_list =
{#- buildout_shared_part_list_dump is ${buildout:shared-part-list} rendered as a
template during software step.
Because it can contain new lines, it's not possible to use it directly when generating
buildout config files from buildout, because the newlines don't get indented and cause
the instance buildout to be invalid ini file.
So we had to dump it in a simple text file and now we parse again that text file. -#}
{%- for line in codecs.open(buildout_shared_part_list_dump).readlines() %}
{{ line.strip() }}
{%- endfor %}
{{ slaprunner['shared_root'] }}
master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
master_rest_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}/hateoas
computer_id = slaprunner
maximal_delay = 0
root_check = {{ slaprunner['root_check'] }}
forbid_supervisord_automatic_launch = true
pidfile_software = {{slaprunner['pidfile-software']}}
pidfile_instance = {{slaprunner['pidfile-instance']}}
[slapformat]
input_definition_file = {{ slaprunner['slapformat-definition.cfg'] }}
partition_amount = {{ slaprunner['partition-amount'] }}
alter_user = false
alter_network = false
create_tap = false
create_tun = false
computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
[slaprunner]
slapos = {{ slaprunner['slapos'] }}
slapos_cfg = {{ slaprunner['slapos.cfg'] }}
slapproxy = {{ slaprunner['slapproxy'] }}
supervisor = {{ slaprunner['supervisor'] }}
supervisord_config = {{ slaprunner['supervisord_config'] }}
supervisord_server = {{ slaprunner['supervisord_server'] }}
runner_workdir = {{ slaprunner['working-directory'] }}
runner_host = {{ slaprunner['ipv4'] }}
runner_port = {{ slaprunner['runner_port'] }}
instance_monitoring_url = {{ slaprunner['instance-monitor-url'] }}
ipv4_address = {{ slaprunner['ipv4'] }}
ipv6_address = {{ slaprunner['ipv6'] }}
etc_dir = {{ slaprunner['etc_dir'] }}
run_dir = {{ slaprunner['run_dir'] }}
log_dir = {{ slaprunner['log_dir'] }}
console = {{ slaprunner['console'] }}
verbose = {{ slaprunner['verbose'] }}
debug = {{ slaprunner['debug'] }}
auto_deploy = {{ slaprunner['auto_deploy'] }}
auto_deploy_instance = {{ slaprunner['auto_deploy_instance'] }}
autorun = {{ slaprunner['autorun'] }}
knowledge0_cfg = {{ slaprunner['knowledge0_file'] }}
minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
minishell_history_file = {{ slaprunner['minishell_history_file'] }}
path = {{ slaprunner['path'] }}
instance_name = {{ slaprunner['instance_name'] }}
default_repository = {{ slaprunner['default_repository'] }}
default_repository_branch = {{ slaprunner['default_repository_branch'] }}
[slapproxy]
host = {{ slaprunner['ipv4'] }}
port = {{ slaprunner['proxy_port'] }}
database_uri = {{ slaprunner['proxy_database'] }}
[gitclient]
git = {{ slaprunner['git-binary'] }}
[sshkeys_authority]
public_key = {{ slaprunner['public_key'] }}
software/slaprunner/template/supervisord.conf.in deleted 100644 → 0
View file @ d2c3e801
[unix_http_server]
file = {{ supervisord['socket_path'] }}
[supervisorctl]
serverurl = {{ supervisord['socket_name'] }}
[inet_http_server]
port = {{ supervisord['server'] }}
[supervisord]
logfile = {{ supervisord['logfile'] }}
pidfile = {{ supervisord['pidfile'] }}
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[program:{{- supervisord['slapgrid-sr'] -}}]
command = {{ supervisord['slapgrid-sr-command'] }}
process_name = {{ supervisord['slapgrid-sr'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapgrid-sr-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
{# how many parallel build jobs to spawn when compiling software -#}
{% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
[program:{{- supervisord['slapgrid-cp'] -}}]
command = {{ supervisord['slapgrid-cp-command'] }}
process_name = {{ supervisord['slapgrid-cp'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapgrid-cp-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapformat'] -}}]
command = {{ supervisord['slapformat-command'] }}
process_name = {{ supervisord['slapformat'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapformat-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapboot'] -}}]
command = {{ supervisord['slapboot-command'] }}
process_name = {{ supervisord['slapboot'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapboot-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapbang'] -}}]
command = {{ supervisord['slapbang-command'] }}
process_name = {{ supervisord['slapbang'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapbang-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapproxy'] -}}]
command = {{ supervisord['slapproxy-command'] }}
process_name = {{ supervisord['slapproxy'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['slapproxy-autostart'] }}
startsecs = {{ supervisord['slapproxy-startsecs'] }}
exitcodes = {{ supervisord['exitcodes'] }}
autorestart = {{ supervisord['slapproxy-autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[eventlistener:listener-slapgrid]
command = {{ listener_slapgrid }}
events = PROCESS_STATE_EXITED
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['stderr_logfile'] }}
software/slaprunner/test/README.md deleted 100644 → 0
View file @ d2c3e801
Tests for slaprunner Software Release
software/slaprunner/test/setup.py deleted 100644 → 0
View file @ d2c3e801
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.slaprunner'
with open("README.md") as f:
long_description = f.read()
setup(name=name,
version=version,
description="Test for SlapOS' slaprunner",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'psutil',
'paramiko',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/slaprunner/test/test.py deleted 100644 → 0
View file @ d2c3e801
This diff is collapsed.
software/theia/buildout.hash.cfg
View file @ fc5a017f
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = ca0f353d00131851a992cd3519bec5b0
md5sum = 4b69822605e3f20fcec8f44a0bdf80bb
[instance]
_update_hash_filename_ = instance.cfg.in
......
software/theia/instance-theia.cfg.jinja.in
View file @ fc5a017f
......@@ -14,7 +14,6 @@ theia-environment-parts =
theia-parts =
frontend-instance
frontend-instance-rsyslogd
python-server
promises
......@@ -98,7 +97,6 @@ instance-promises =
{% if additional_frontend %}
$${remote-additional-frontend-url-available-promise:name}
{% endif %}
$${frontend-instance-rsyslogd-promise:name}
$${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name}
......@@ -153,12 +151,6 @@ config-url = $${remote-additional-frontend:connection-secure_access}
config-http-code = 401
{% endif %}
[frontend-instance-rsyslogd-promise]
<= monitor-promise-base
promise = check_command_execute
name = rsyslogd_listen_promise.py
config-command = test -S $${frontend-instance-rsyslogd-config:log-socket}
[slapos-standalone-listen-promise]
<= monitor-promise-base
promise = check_socket_listening
......@@ -232,39 +224,6 @@ recipe = slapos.cookbook:generate.password
username = admin
storage-path = $${buildout:parts-directory}/.$${:_buildout_section_name_}
[frontend-instance-rsyslogd-config]
recipe = slapos.recipe.template
output = $${directory:etc}/$${:_buildout_section_name_}
log-file = $${directory:log}/frontend-instance.log
log-socket = $${directory:run}/rsyslog.sock
pidfile = $${directory:pidfiles}/rsyslogd.pid
inline =
module(
load="imuxsock"
SysSock.Name="$${:log-socket}")
# Just simply output the raw line without any additional information, as
# haproxy emits enough information by itself
# Also cut out first empty space in msg, which is related to rsyslogd
# internal and end up cutting on 8k, as it's default of $MaxMessageSize
template(name="rawoutput" type="string" string="%msg:2:8192%\n")
$ActionFileDefaultTemplate rawoutput
$FileCreateMode 0600
$DirCreateMode 0700
$Umask 0022
$WorkDirectory $${directory:run}
*.* $${:log-file};rawoutput
[frontend-instance-rsyslogd]
recipe = slapos.cookbook:wrapper
command-line = ${rsyslogd:location}/sbin/rsyslogd -i $${frontend-instance-rsyslogd-config:pidfile} -n -f $${frontend-instance-rsyslogd-config:output}
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
hash-files = $${frontend-instance-rsyslogd-config:output}
[frontend-instance-port]
recipe = slapos.cookbook:free_port
minimum = 3000
......@@ -278,7 +237,7 @@ command =
then
${openssl-output:openssl} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=$${:common-name}" \
-newkey rsa:1024 -keyout $${:cert-file} \
-newkey rsa:2048 -keyout $${:cert-file} \
-out $${:cert-file}
fi
update-command = $${:command}
......@@ -288,34 +247,13 @@ location =
$${:cert-file}
[frontend-instance-config]
recipe = slapos.recipe.template
recipe = slapos.recipe.template:jinja2
url = ${stack-haproxy-default-backend-config:target}
output = $${directory:etc}/$${:_buildout_section_name_}
blankline =
inline =
global
maxconn 4096
master-worker
pidfile $${frontend-instance:pidfile}
log $${frontend-instance-rsyslogd-config:log-socket} local0 info
defaults
log global
option httplog
mode http
retries 1
option redispatch
maxconn 2000
balance roundrobin
timeout connect 10s
timeout queue 60s
timeout server 305s
timeout client 305s
# compress some content types
compression algo gzip
compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
context =
key pidfile frontend-instance:pidfile
key content :content
content =
userlist basic-auth-list
user $${frontend-instance-password:username} insecure-password $${frontend-instance-password:passwd}
......@@ -340,7 +278,6 @@ inline =
backend static
log global
server static_backend $${python-server-port:ip}:$${python-server-port:port}
$${:blankline}
ip = $${frontend-instance-port:ip}
hostname = [$${:ip}]
......
software/theia/software.cfg
View file @ fc5a017f
[buildout]
extends =
../../component/haproxy/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/git/buildout.cfg
../../component/bash/buildout.cfg
../../component/bash-completion/buildout.cfg
......@@ -14,6 +12,7 @@ extends =
../../component/fonts/buildout.cfg
../../component/theia/buildout.cfg
../../component/pygolang/buildout.cfg
../../stack/haproxy/default-backend.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../stack/resilient/buildout.cfg
......
stack/haproxy/buildout.hash.cfg 0 → 100644
View file @ fc5a017f
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[stack-haproxy-default-backend-config]
filename = default-backend.cfg.in
md5sum = dd336cb9ea7dcee94127a994bbe0a2e8
stack/haproxy/default-backend.cfg 0 → 100644
View file @ fc5a017f
[buildout]
extends =
buildout.hash.cfg
../../component/haproxy/buildout.cfg
[stack-haproxy-default-backend-config]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
stack/haproxy/default-backend.cfg.in 0 → 100644
View file @ fc5a017f
global
maxconn 4096
master-worker
pidfile {{ pidfile }}
log stdout format raw daemon
defaults
log global
option httplog
mode http
retries 1
option redispatch
maxconn 2000
balance roundrobin
timeout connect 10s
timeout queue 60s
timeout server 305s
timeout client 305s
compression algo gzip
compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
{{ content }}
{# BLANKLINE #}
stack/monitor/README.md
View file @ fc5a017f
......@@ -78,8 +78,8 @@ If you have sub-instances, you should collect the base monitor url from all inst
Also, all monitors of the sub instances need to have same password as the password of the root instance monitor.
NB: You should use double $ (ex: $${monitor-template:output}) instead of one $ in your instance template file if it's not a jinja template. See:
- Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance-resilient-test.cfg.jinja2
- Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance.cfg
- Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/theia/instance-resilient.cfg.jinja
- Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/theia/instance.cfg.in
Add a promise
-------------
......
stack/slapos.cfg
View file @ fc5a017f
......@@ -285,7 +285,7 @@ simplegeneric = 0.8.1
singledispatch = 3.4.0.3
six = 1.16.0
slapos.cookbook = 1.0.297
slapos.core = 1.8.5
slapos.core = 1.8.6
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.25
slapos.rebootstrap = 4.5
......@@ -346,6 +346,8 @@ smmap2 = 2.0.5
traitlets = 4.3.3
Werkzeug = 1.0.1
wheel = 0.35.1:whl
# for slapos.core
selectors34 = 1.2
[versions:sys.version_info < (3,8)]
MarkupSafe = 1.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7