Commit 1ac1878b authored by Cédric Le Ninivin's avatar Cédric Le Ninivin

Runner: Merge development into master for release

Conflicts:
	software/slaprunner/schema.json
	software/slaprunner/software.cfg.json
parents 4e244f48 089a201f
No related merge requests found
- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance - resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
- shellinabox password should be the same in all the resilient instances - shellinabox password should be the same in all the resilient instances
- add test for parameter auto-deploy-instance - add test for parameter auto-deploy-instance
- Add download facility in file browser
...@@ -43,7 +43,7 @@ mode = 0644 ...@@ -43,7 +43,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in output = ${buildout:directory}/template-runner.cfg.in
md5sum = b5644037da14e79576840975415ee1a4 md5sum = a4f3127bc82ff3a671a65a150f26c969
mode = 0644 mode = 0644
[template-runner-import-script] [template-runner-import-script]
...@@ -103,7 +103,7 @@ mode = 0644 ...@@ -103,7 +103,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/httpd_conf.in url = ${:_profile_base_location_}/httpd_conf.in
download-only = true download-only = true
md5sum = 444399390eb19c7e23dd06593c067b4d md5sum = 94bc5d6f1afc4e075d530fed496f47c2
filename = httpd_conf.in filename = httpd_conf.in
mode = 0644 mode = 0644
...@@ -118,7 +118,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -118,7 +118,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-slapos-cfg] [template-slapos-cfg]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename} url = ${:_profile_base_location_}/template/${:filename}
md5sum = ba3721425ba34e265b1f8390ed7196cf md5sum = ca57a8bf1d4da20344fab602a69b807f
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = slapos.cfg.in filename = slapos.cfg.in
download-only = true download-only = true
...@@ -145,7 +145,7 @@ mode = 0644 ...@@ -145,7 +145,7 @@ mode = 0644
[template-supervisord] [template-supervisord]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename} url = ${:_profile_base_location_}/template/${:filename}
md5sum = af2d78440a358ec268d7ece4fce99ec9 md5sum = 239fe88e1522afe7686572dd34568720
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = supervisord.conf.in filename = supervisord.conf.in
download-only = true download-only = true
...@@ -160,6 +160,15 @@ filename = listener_slapgrid.py.in ...@@ -160,6 +160,15 @@ filename = listener_slapgrid.py.in
download-only = true download-only = true
mode = 0644 mode = 0644
[cors-domain-cgi]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
download-only = true
md5sum = 4c44d617d5bfd1db8695200e896480c0
destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi
filename = cors-domain.jinja
mode = 0644
[eggs] [eggs]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
eggs = eggs =
...@@ -205,7 +214,3 @@ recipe = zc.recipe.egg ...@@ -205,7 +214,3 @@ recipe = zc.recipe.egg
interpreter = pythonwitheggs interpreter = pythonwitheggs
eggs += eggs +=
supervisor supervisor
[versions]
# We still need to use the older version.
slapos.core = 1.0.5
...@@ -27,6 +27,7 @@ LoadModule dav_module modules/mod_dav.so ...@@ -27,6 +27,7 @@ LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule cache_module modules/mod_cache.so LoadModule cache_module modules/mod_cache.so
LoadModule file_cache_module modules/mod_file_cache.so LoadModule file_cache_module modules/mod_file_cache.so
LoadModule setenvif_module modules/mod_setenvif.so
ErrorLog "{{ parameters.path_error_log }}" ErrorLog "{{ parameters.path_error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b" common LogFormat "%h %l %u %t \"%r\" %>s %b" common
...@@ -45,6 +46,11 @@ SSLHonorCipherOrder On ...@@ -45,6 +46,11 @@ SSLHonorCipherOrder On
SSLCipherSuite RC4-SHA:HIGH:!ADH SSLCipherSuite RC4-SHA:HIGH:!ADH
SSLEngine On SSLEngine On
Include {{ parameters.httpd_cors_file }}
Header set Access-Control-Allow-Credentials "true"
Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
Alias /public {{ parameters.runner_home }}/public Alias /public {{ parameters.runner_home }}/public
<Directory {{ parameters.runner_home }}/public> <Directory {{ parameters.runner_home }}/public>
Order Allow,Deny Order Allow,Deny
...@@ -69,7 +75,9 @@ Alias /share {{ parameters.runner_home }} ...@@ -69,7 +75,9 @@ Alias /share {{ parameters.runner_home }}
AuthType Basic AuthType Basic
AuthName "webdav" AuthName "webdav"
AuthUserFile "{{ parameters.etc_dir }}/.htpasswd" AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
Require valid-user <LimitExcept OPTIONS>
Require valid-user
</LimitExcept>
</Directory> </Directory>
ScriptSock {{ parameters.path_pid }} ScriptSock {{ parameters.path_pid }}
......
...@@ -91,6 +91,11 @@ ...@@ -91,6 +91,11 @@
"minimum": 9683, "minimum": 9683,
"exclusiveMinimum": true, "exclusiveMinimum": true,
}, },
"cpu-usage-ratio": {
"description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
"type": "integer",
"default" : 4
},
"resilient-clone-number": { "resilient-clone-number": {
"title": "Number of Resilient Clones", "title": "Number of Resilient Clones",
"description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2", "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Runner instanciation",
"properties": {
"backend_url": {
"description": "ipv6 url to directly access your runner",
"type": "string"
},
"access_url": {
"description": "Url to set up your account and then login into your webrunner",
"type": "string"
},
"url": {
"description": "Url to access your runner once you set your account",
"type": "string"
},
"ssh_command": {
"description": "SSH command used to access your runner in ssh when you provided a ssh public key"
"type": "string"
},
"monitor_url": {
"description": "Url to access the monitoring interface of your instance. Use this url to get the recovery code needed to set up your runner",
"type": "string"
},
"webdav_url": {
"description": "Url for webdav access of your runner it access the root of your runner. Same users as the one for the runner",
"type": "string"
},
"public_url": {
"description": "Url of the public folder of your runner",
"type": "string"
},
"git_public_url": {
"description": "Public url of your repositories.",
"type": "string"
},
"git_private_url": {
"description": "Private url for your repositories. You can use it to push.",
"type": "string"
}
},
"type": "object"
}
\ No newline at end of file
...@@ -50,6 +50,7 @@ parts = ...@@ -50,6 +50,7 @@ parts =
monitor-instance-log-access monitor-instance-log-access
## Monitor for runner ## Monitor for runner
monitor-current-log-access monitor-current-log-access
monitor-deploy-cors-domain-cgi
extends = ${monitor-template:output} extends = ${monitor-template:output}
...@@ -190,6 +191,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd ...@@ -190,6 +191,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd
minishell_history_file = $${directory:etc}/.minishell_history minishell_history_file = $${directory:etc}/.minishell_history
software_info_json = $${runnerdirectory:home}/software_info.json software_info_json = $${runnerdirectory:home}/software_info.json
instance_info_json = $${runnerdirectory:home}/instance_info.json instance_info_json = $${runnerdirectory:home}/instance_info.json
path = $${shell:path}
[test-runner] [test-runner]
<= slaprunner <= slaprunner
...@@ -207,7 +209,7 @@ auto_deploy = True ...@@ -207,7 +209,7 @@ auto_deploy = True
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${buildout:directory}/bin/slaprunnertest command-line = ${buildout:directory}/bin/slaprunnertest
wrapper-path = $${directory:bin}/runTestSuite wrapper-path = $${directory:bin}/runTestSuite
environment = RUNNER_CONFIG=$${slapos-test-cfg:rendered} environment = RUNNER_CONFIG=$${slapos-cfg:rendered}
# Deploy dropbear (minimalist SSH server) # Deploy dropbear (minimalist SSH server)
[sshkeys-directory] [sshkeys-directory]
...@@ -328,6 +330,7 @@ project_public_folder = $${runnerdirectory:public-project} ...@@ -328,6 +330,7 @@ project_public_folder = $${runnerdirectory:public-project}
runner_home = $${runnerdirectory:home} runner_home = $${runnerdirectory:home}
git_http_backend = ${git:location}/libexec/git-core/git-http-backend git_http_backend = ${git:location}/libexec/git-core/git-http-backend
cgi_httpd_conf = $${monitor-httpd-configuration-file:rendered} cgi_httpd_conf = $${monitor-httpd-configuration-file:rendered}
httpd_cors_file = $${monitor-httpd-cors:location}
[httpd-conf] [httpd-conf]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -642,12 +645,12 @@ pidfile = $${directory:run}/supervisord.pid ...@@ -642,12 +645,12 @@ pidfile = $${directory:run}/supervisord.pid
server = $${slaprunner:ipv4}:$${:port} server = $${slaprunner:ipv4}:$${:port}
port = 39986 port = 39986
slapgrid-cp = slapgrid-cp slapgrid-cp = slapgrid-cp
slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${slaprunner:slapos.cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log} slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log}
slapgrid-cp-log = $${runnerdirectory:home}/instance.log slapgrid-cp-log = $${runnerdirectory:home}/instance.log
slapgrid-cp-pid = $${directory:run}/slapgrid-cp.pid slapgrid-cp-pid = $${directory:run}/slapgrid-cp.pid
slapgrid-cp-startretries = 0 slapgrid-cp-startretries = 0
slapgrid-sr = slapgrid-sr slapgrid-sr = slapgrid-sr
slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${slaprunner:slapos.cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log} slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log}
slapgrid-sr-log = $${runnerdirectory:home}/software.log slapgrid-sr-log = $${runnerdirectory:home}/software.log
slapgrid-sr-pid = $${directory:run}/slapgrid-sr.pid slapgrid-sr-pid = $${directory:run}/slapgrid-sr.pid
slapgrid-sr-startretries = 0 slapgrid-sr-startretries = 0
...@@ -655,7 +658,7 @@ slapproxy = slapproxy ...@@ -655,7 +658,7 @@ slapproxy = slapproxy
slapproxy-autorestart = true slapproxy-autorestart = true
slapproxy-autostart = true slapproxy-autostart = true
slapproxy-startsecs = 1 slapproxy-startsecs = 1
slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${slaprunner:slapos.cfg} slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
slapproxy-log = $${directory:log}/slapproxy.log slapproxy-log = $${directory:log}/slapproxy.log
socket_name = unix://$${:socket_path} socket_name = unix://$${:socket_path}
socket_path = $${directory:tmp}/supervisord.sock socket_path = $${directory:tmp}/supervisord.sock
...@@ -663,13 +666,17 @@ startsecs = 0 ...@@ -663,13 +666,17 @@ startsecs = 0
# This file logs errors from listeners. Supervisord has its own logfile. # This file logs errors from listeners. Supervisord has its own logfile.
# Processes should handle their logs by themselves # Processes should handle their logs by themselves
stderr_logfile = $${directory:log}/supervisord-errors.log stderr_logfile = $${directory:log}/supervisord-errors.log
slapos-cfg = $${slaprunner:slapos.cfg}
[supervisord-conf] [supervisord-conf]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-supervisord:location}/${template-supervisord:filename} template = ${template-supervisord:location}/${template-supervisord:filename}
rendered = $${directory:etc}/supervisord.conf rendered = $${directory:etc}/supervisord.conf
context = context =
import multiprocessing multiprocessing
import builtin __builtin__
section supervisord supervisord section supervisord supervisord
key slapparameter_dict slap-configuration:configuration
key listener_slapgrid listener-slapgrid-bin:rendered key listener_slapgrid listener-slapgrid-bin:rendered
[listener-slapgrid-bin] [listener-slapgrid-bin]
...@@ -696,3 +703,25 @@ port = $${supervisord:port} ...@@ -696,3 +703,25 @@ port = $${supervisord:port}
[monitor-current-log-access] [monitor-current-log-access]
< = monitor-directory-access < = monitor-directory-access
source = $${directory:log} source = $${directory:log}
[monitor-deploy-cors-domain-cgi]
recipe = slapos.recipe.template:jinja2
template = ${cors-domain-cgi:location}/${cors-domain-cgi:filename}
rendered = $${monitor-directory:knowledge0-cgi}/$${:filename}
filename = cors-domain.cgi
mode = 0744
context =
raw config_cfg $${buildout:directory}/knowledge0.cfg
raw timestamp $${buildout:directory}/.timestamp
raw python_executable ${buildout:executable}
key apache_file httpd-parameters:httpd_cors_file
key pwd monitor-directory:knowledge0-cgi
key this_file :filename
key httpd_graceful cgi-httpd-graceful-wrapper:rendered
[monitor-httpd-cors]
recipe = plone.recipe.command
command = if [ ! -f $${:location} ]; then touch $${:location}; fi
location = $${directory:etc}/$${:filename}
filename = httpd-cors.cfg
stop-on-error = true
...@@ -14,6 +14,7 @@ async = 0.6.1 ...@@ -14,6 +14,7 @@ async = 0.6.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
collective.recipe.template = 1.11 collective.recipe.template = 1.11
ecdsa = 0.11
erp5.util = 0.4.41 erp5.util = 0.4.41
gitdb = 0.5.4 gitdb = 0.5.4
gunicorn = 19.1.1 gunicorn = 19.1.1
...@@ -22,26 +23,22 @@ pycrypto = 2.6.1 ...@@ -22,26 +23,22 @@ pycrypto = 2.6.1
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.build = 0.12 slapos.recipe.build = 0.12
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.40.1 slapos.toolbox = 0.40.2
smmap = 0.8.2 smmap = 0.8.2
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
# Required by: # Required by:
# slapos.toolbox==0.39.3 # slapos.toolbox==0.40.2
GitPython = 0.3.2.RC1 GitPython = 0.3.2.RC1
# Required by: # Required by:
# slapos.toolbox==0.39.3 # slapos.toolbox==0.40.2
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# paramiko==1.14.1 # slapos.toolbox==0.40.2
ecdsa = 0.11
# Required by:
# slapos.toolbox==0.39.3
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# slapos.toolbox==0.39.3 # slapos.toolbox==0.40.2
paramiko = 1.14.1 paramiko = 1.15.1
\ No newline at end of file
#!{{ python_executable }}
import cgi
import cgitb
import ConfigParser
import os
import re
import subprocess
cgitb.enable(display=0, logdir="/tmp/cgi.log")
form = cgi.FieldStorage()
apache_file = "{{ apache_file }}"
config_file = "{{ config_cfg }}"
if not os.path.exists(config_file):
print "Your software does <b>not</b> embed 0-knowledge. \
This interface is useless in this case</body></html>"
exit(0)
parser = ConfigParser.ConfigParser()
parser.read(config_file)
if not parser.has_section("cors"):
parser.add_section("cors")
if not parser.has_option("cors", "cors-domain"):
parser.set("cors", "cors-domain", "")
if "cors-domain" in form:
parser.set("cors", "cors-domain", form["cors-domain"].value)
cors_domain_list = form["cors-domain"].value.split()
cors_string = ""
for domain in cors_domain_list:
if cors_string:
cors_string += '|'
cors_string += re.escape(domain)
with open(apache_file, 'w') as file:
file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
subprocess.call('{{ httpd_graceful }}')
cors_domain_parameter = parser.get("cors", "cors-domain")
with open(config_file, 'w') as file:
parser.write(file)
print "<html><head>"
print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"static/style.css\">"
print "</head><body>"
print "<h1>Cors Domains :</h1>"
print "<p>Enter domain names separated by space to authorized CORS on your dav storage for these domains</p>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
print "<div class=\"pure-control-group\">"
print '<label for="cors-domain">CORS Domains</label>'
print '<input type="text" name="cors-domain" value="%s">' % cors_domain_parameter
print "</div>"
print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
pure-button-primary\">Save</button></div></form>"
print "</body></html>"
...@@ -34,6 +34,7 @@ autorun = {{ slaprunner['autorun'] }} ...@@ -34,6 +34,7 @@ autorun = {{ slaprunner['autorun'] }}
knowledge0_cfg = {{ slaprunner['knowledge0_file'] }} knowledge0_cfg = {{ slaprunner['knowledge0_file'] }}
minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }} minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
minishell_history_file = {{ slaprunner['minishell_history_file'] }} minishell_history_file = {{ slaprunner['minishell_history_file'] }}
path = {{ slaprunner['path'] }}
[slapproxy] [slapproxy]
host = {{ slaprunner['ipv4'] }} host = {{ slaprunner['ipv4'] }}
......
...@@ -26,7 +26,7 @@ autorestart = {{ supervisord['autorestart'] }} ...@@ -26,7 +26,7 @@ autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }} stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }} stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }} directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}" environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{- '%d' % builtin.max(1, (multiprocessing.cpu_count() / builtin.int(slapparameter_dict.get('cpu-usage-ration', 4)))) -}}"
[program:{{- supervisord['slapgrid-cp'] -}}] [program:{{- supervisord['slapgrid-cp'] -}}]
command = {{ supervisord['slapgrid-cp-command'] }} command = {{ supervisord['slapgrid-cp-command'] }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment