Commit cd3cc2d8 authored by Rafael Monnerat's avatar Rafael Monnerat

Remove git server and add slapos format to work on webrunner

See merge request !747
parents 73c19b03 38ac3e3d
No related merge requests found
......@@ -139,17 +139,6 @@ Example :
<parameter id="custom-frontend-basic-auth">true</parameter>
</instance>
Git repositories :
------------------
It is easy to give access to your git repository/ies to everyone, or to clone it on your own computer. For this, fetch the git_url parameter returned by your webrunner instance. It should look like that : https://[IPV6]:PORT/git/YourRepo.git/
To create the repo, go to the folder srv/runner/project and initiate a new git repo (git init/clone --bare YourRepo). Then, to clone the repo, just append its name to the previous URL.
Please notice that your repositories are in public access for reading. However, an account is needed to push to the repository (use your webrunner account, or create a new user in the parameters section of the webrunner).
For the moment, the PORT is the port of monitoring, which is 9685.
Things to notice for the nex developer :
----------------------------------------
......
......@@ -18,11 +18,11 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 7100c6678576c285322f1445029cd4f2
md5sum = 00915c7ad17e335c87be40bae4bd0cfb
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = 2852689640b632fc713d7a603ce66207
md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
[instance-runner-import]
filename = instance-runner-import.cfg.in
......@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
[template-resilient]
filename = instance-resilient.cfg.jinja2
md5sum = 105ed7f54f251c64a2d34559360a5604
md5sum = bd0ad0b80d2b39189f9665c48f1b3830
[template_nginx_conf]
filename = nginx_conf.in
......@@ -42,7 +42,7 @@ md5sum = 2b06f7eb9a1d45d250d4b92a944db925
[template_httpd_conf]
filename = httpd_conf.in
md5sum = 40825ef2d259ae3b86dfd2f28f597d3a
md5sum = 1d96c29b931383b2a87b9ae049d07d74
[template_launcher]
filename = launcher.in
......@@ -50,7 +50,11 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
[template-slapos-cfg]
filename = template/slapos.cfg.in
md5sum = e6a3ca1604ae5458248135cd6de0f3e6
md5sum = 95de0677e78fc06cc8304cc6caea9169
[template-slapformat-definition.cfg]
filename = template/slapformat-definition.cfg.in
md5sum = a94299dd2133956060486609e30b97ec
[template-parameters]
filename = parameters.xml.in
......@@ -62,7 +66,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
[template-supervisord]
filename = template/supervisord.conf.in
md5sum = 28f69b57c2835bddfcfbe312563eef51
md5sum = 37f053d75752e998fc3bb9e4bf29d776
[template-listener-slapgrid]
filename = template/listener_slapgrid.py.in
......
......@@ -96,33 +96,3 @@ Alias /share {{ parameters.runner_home }}
</LimitExcept>
</Directory>
SetEnv GIT_HTTP_EXPORT_ALL
ScriptSock {{ parameters.cgid_sock }}
ScriptAlias /git/ {{ parameters.git_http_backend }}/
ScriptAlias /git-public/ {{ parameters.git_http_backend }}/
RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
RewriteCond %{REQUEST_URI} /git-receive-pack$
<LocationMatch "^/git/">
SetEnv GIT_PROJECT_ROOT {{ parameters.project_private_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.htpasswd_file }}"
Require valid-user
</LocationMatch>
<LocationMatch "^/git-public/">
SetEnv GIT_PROJECT_ROOT {{ parameters.project_public_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.htpasswd_file }}"
Require valid-user
Satisfy any
</LocationMatch>
......@@ -12,7 +12,7 @@
{% set number_of_instances = 2 %}
{% endif %}
{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url', 'git-public-url', 'git-private-url'] -%}
{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url'] -%}
{% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
{% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
{% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
......@@ -72,8 +72,6 @@ init-password = ${publish-early:init-password}
ssh-command = ${request-runner:connection-ssh-command}
webdav-url = ${request-runner:connection-webdav-url}
public-url = ${request-runner:connection-public-url}
git-public-url = ${request-runner:connection-git-public-url}
git-private-url = ${request-runner:connection-git-private-url}
{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
custom-frontend-url = ${request-runner:connection-custom-frontend-url}
{% endif %}
......
......@@ -29,14 +29,6 @@
"public_url": {
"description": "Url of the public folder of your runner",
"type": "string"
},
"git_public_url": {
"description": "Public url of your repositories.",
"type": "string"
},
"git_private_url": {
"description": "Private url for your repositories. You can use it to push.",
"type": "string"
}
},
"type": "object"
......
......@@ -21,6 +21,7 @@ common-runner-parts =
shellinabox
shellinabox-service
slapos-cfg
slapformat-definition.cfg
cron-entry-prepare-software
deploy-instance-parameters
instance-software
......@@ -151,8 +152,6 @@ project-test = $${:test}/project
software-test = $${:test}/software
instance-test = $${:test}/instance
sessions = $${buildout:directory}/.sessions
private-project = $${:home}/.git-private
public-project = $${:home}/.git-public
[slaprunner]
slaprunner = ${buildout:bin-directory}/slaprunner
......@@ -162,6 +161,7 @@ supervisor = ${buildout:bin-directory}/slapgrid-supervisorctl
git-binary = ${git:location}/bin/git
root_check = false
slapos.cfg = $${directory:etc}/slapos.cfg
slapformat-definition.cfg = $${directory:etc}/slapformat-definition.cfg
working-directory = $${runnerdirectory:home}
project-directory = $${runnerdirectory:project}
instance_root = $${runnerdirectory:instance-root}
......@@ -389,11 +389,7 @@ htpasswd_file = $${directory:etc}/.htpasswd
etc_dir = $${directory:etc}
var_dir = $${directory:var}
project_folder = $${directory:project}
project_private_folder = $${runnerdirectory:private-project}
project_public_folder = $${runnerdirectory:public-project}
runner_home = $${runnerdirectory:home}
git_http_backend = ${git:location}/libexec/git-core/git-http-backend
cgid_sock = $${directory:run}/cgid.sock
httpd_cors_file = $${slaprunner-httpd-cors:location}
[httpd-conf]
......@@ -591,8 +587,6 @@ init-user = $${runner-htpasswd:user}
init-password = $${runner-htpasswd:password}
ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshd-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' -%}
url = https://$${request-frontend:connection-domain}
......@@ -672,6 +666,17 @@ mode = 700
context =
section slaprunner test-runner
[slapformat-definition.cfg]
recipe = slapos.recipe.template:jinja2
template = ${template-slapformat-definition.cfg:location}/${template-slapformat-definition.cfg:filename}
rendered = $${slaprunner:slapformat-definition.cfg}
mode = 700
context =
section slaprunner slaprunner
key partition_user user-info:pw-name
import codecs codecs
raw buildout_shared_part_list_dump $${slaprunner:buildout-shared-part-list-dump}
[shellinabox]
recipe = slapos.recipe.template:jinja2
# We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
......@@ -785,6 +790,10 @@ slapproxy-autostart = true
slapproxy-startsecs = 1
slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
slapproxy-log = $${directory:log}/slapproxy.log
slapformat = slapformat
slapformat-command = $${slaprunner:slapos} node format --cfg $${:slapos-cfg} --verbose --logfile $${:slapformat-log} --now -i $${:slapformat-definition.cfg}
slapformat-log = $${directory:log}/slapos-node-format.log
slapformat-startretries = 0
socket_name = unix://$${:socket_path}
socket_path = $${directory:tmp}/supervisord.sock
startsecs = 0
......@@ -792,6 +801,7 @@ startsecs = 0
# Processes should handle their logs by themselves
stderr_logfile = $${directory:log}/supervisord-errors.log
slapos-cfg = $${slaprunner:slapos.cfg}
slapformat-definition.cfg = $${slaprunner:slapformat-definition.cfg}
[supervisord-conf]
recipe = slapos.recipe.template:jinja2
......
......@@ -42,6 +42,7 @@ common-parts =
instance-runner-import
instance-runner-export
template-slapos-cfg
template-slapformat-definition.cfg
template-slapuser-script
# XXX: we have to manually add this for resilience
rdiff-backup
......@@ -110,6 +111,10 @@ recipe = slapos.recipe.build:download
< = template-download-base
filename = slapos.cfg.in
[template-slapformat-definition.cfg]
< = template-download-base
filename = slapformat-definition.cfg.in
[template-parameters]
< = download-only-base
......
......@@ -173,7 +173,7 @@ if 1:
# We want to replace the last occurence only
old_software_release, partition = sys.argv[1], sys.argv[2]
for match in re.finditer("(slappart|test0-)[0-9][0-9]*", old_software_release):
for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
start, end = match.start(), match.end()
print old_software_release[:start] + partition + old_software_release[end:]
......
[computer]
# Address example: 2001:67c:1254:27::1/ffff:ffff:ffff:ffff::/64
address = {{ slaprunner['ipv6'] }}/64
{% for partition_index in range(int(slaprunner['partition-amount'])) %}
[partition_{{ partition_index }}]
address = {{ slaprunner['ipv4'] }}/255.255.255.255 {{ slaprunner['ipv6'] }}/64
user = {{ partition_user }}
pathname = slappart{{ partition_index }}
network_interface =
{% endfor %}
......@@ -22,6 +22,11 @@ pidfile_instance = {{slaprunner['pidfile-instance']}}
[slapformat]
partition_amount = {{ slaprunner['partition-amount'] }}
alter_user = false
alter_network = false
create_tab = false
create_tun = false
computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
[slaprunner]
slapos = {{ slaprunner['slapos'] }}
......
......@@ -44,6 +44,20 @@ stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapformat'] -}}]
command = {{ supervisord['slapformat-command'] }}
process_name = {{ supervisord['slapformat'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapformat-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapproxy'] -}}]
command = {{ supervisord['slapproxy-command'] }}
process_name = {{ supervisord['slapproxy'] }}
......
......@@ -49,7 +49,6 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
class SlaprunnerTestCase(SlapOSInstanceTestCase):
# Slaprunner uses unix sockets, so it needs short paths.
__partition_reference__ = 's'
......@@ -106,26 +105,6 @@ class TestWeb(SlaprunnerTestCase):
self.assertEqual(requests.codes.ok, hello.status_code)
self.assertIn('<b>Hello</b>', hello.text)
# git seems broken, these are 404 now...
@unittest.expectedFailure
def test_git_private(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-private']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
resp = requests.get(
url,
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
@unittest.expectedFailure
def test_git_public(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-public']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
class TestSSH(SlaprunnerTestCase):
@classmethod
......@@ -275,6 +254,7 @@ class ServicesTestCase(SlaprunnerTestCase):
class TestInstanceResilient(SlaprunnerTestCase):
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
......@@ -287,8 +267,6 @@ class TestInstanceResilient(SlaprunnerTestCase):
'backend-url',
'feed-url-runner-1-pull',
'feed-url-runner-1-push',
'git-private-url',
'git-public-url',
'init-password',
'init-user',
'monitor-base-url',
......@@ -308,7 +286,7 @@ class TestCustomFrontend(SlaprunnerTestCase):
'custom-frontend-backend-url': 'https://www.erp5.com',
'custom-frontend-backend-type': 'redirect',
}
def test(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
# slapproxy returns the backend URL when requesting a slave frontend
......
......@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
[resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in
md5sum = 7a0f1be7e9d05d0c68748322abe1f511
md5sum = 4137b7ac6b20a6cd897ad58f51a763b1
[template-wrapper]
filename = templates/wrapper.in
......
......@@ -48,7 +48,7 @@ def getLatestBackupDate():
if not db_keys:
result = False
else:
last_backup = db[db_keys[0]]
last_backup = float(db[db_keys[0]])
for callback in db_keys:
timestamp = float(db[callback])
if timestamp > last_backup:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment