Commit 6d65a74e authored by Alain Takoudjou's avatar Alain Takoudjou

Merge branch 'kvm-cluster'

parents a7cf2ea1 30583b7f
[buildout]
extends =
../curl/buildout.cfg
../openssl/buildout.cfg
parts =
pycurl
[pycurl-env]
PATH = ${curl:location}/bin:${openssl:location}/bin:%(PATH)s
PYCURL_SSL_LIBRARY=openssl
CPPFLAGS=-I${openssl:location}/include
CFLAGS=-I${openssl:location}/include
[pycurl]
recipe = zc.recipe.egg:custom
egg = pycurl
rpath =
${curl:location}/lib/
${openssl:location}/lib/
environment = pycurl-env
......@@ -12,9 +12,11 @@ extends =
../../component/openssl/buildout.cfg
../../component/dcron/buildout.cfg
../../component/netcat/buildout.cfg
../../component/pycurl/buildout.cfg
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../stack/resilient/buildout.cfg
../../stack/monitor/buildout.cfg
# stacks are listed from most generic to most specific,
# to avoid versioning issues
......@@ -51,6 +53,7 @@ eggs =
cns.recipe.symlink
collective.recipe.template
plone.recipe.command
${pycurl:egg}
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
......@@ -87,7 +90,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 3bca2c959d19881270c64f94ad1ebba8
md5sum = 41eb12b66bd1af86b1b98b2166e86417
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -95,7 +98,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = ea1e8f4a7c1878beec83267fd40728c2
md5sum = da3b3bde94ac2c06759a89c78168df91
download-only = true
on-update = true
......@@ -103,7 +106,7 @@ on-update = true
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = 5a864099760e3a37fa4604044d708657
md5sum = 6ac9d1b06398b57cc1c8d63a9c492415
download-only = true
on-update = true
......@@ -168,12 +171,20 @@ md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
output = ${buildout:directory}/template-frontend.cfg
mode = 0644
[template-ansible-promise]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/ansible-promise.in
md5sum = d0bd07d5a7799b7aea7720ffdf46f322
mode = 0644
download-only = true
filename = ansible-promise.in
[template-apache-conf]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/apache.conf.in
mode = 644
filename = apache.conf.in
md5sum = e9c9fd88d71e9dc7416149af5bcfb951
md5sum = 355fdabdb86fee8e9714b6d357149958
download-only = true
on-update = true
......@@ -186,6 +197,17 @@ md5sum = 822737e483864bf255ad1259237bef2a
download-only = true
on-update = true
[template-logrotate-base]
recipe = slapos.recipe.template:jinja2
filename = instance-logrotate-base.cfg
template = ${:_profile_base_location_}/instance-logrotate-base.cfg.in
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-logrotate-base.cfg
md5sum = f28fbd310944f321ccb34b2a34c82005
context =
key dcron_location dcron:location
key gzip_location gzip:location
key logrotate_location logrotate:location
[template-httpd]
recipe = slapos.recipe.template:jinja2
filename = template-httpd.cfg
......
......@@ -294,6 +294,12 @@
"title": "Text content to send to this virtual machine.",
"description": "Text content which will be written in a file 'data' of http server of this virtual machine instance. The file will be available via URL: http://10.0.2.100/data in the VM.",
"type": "string"
},
"enable-monitor": {
"title": "Deploy monitoring tools",
"description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
"type": "boolean",
"default": true
}
},
"type": "object"
......
......@@ -64,6 +64,7 @@ config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
{% if kvm_parameter_dict.get('data-to-vm', '') -%}
config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
{% endif -%}
config-enable-monitor = True
# Enable simple http server on ipv6 so all VMs will access it
config-document-host = ${apache-conf:ip}
......
......@@ -135,6 +135,12 @@
"description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
"type": "string"
},
"enable-monitor": {
"title": "Deploy monitoring tools",
"description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
"type": "boolean",
"default": true
},
"enable-http-server": {
"title": "Enable local http server",
"description": "Set if local http server which serve files to the vm should be deployed. If set to true, get file into the vm with URL: http://10.0.2.100/FILE.",
......
{% set enable_http = slapparameter_dict.get('enable-http-server', 'False').lower() -%}
{% set use_tap = slapparameter_dict.get('use-tap', 'False').lower() -%}
{% set use_nat = slapparameter_dict.get('use-nat', 'True').lower() -%}
{% set name = slapparameter_dict.get('name', 'localhost') -%}
{% set monitor = slapparameter_dict.get('enable-monitor', 'True').lower() -%}
{% set frontend_software_type = 'default' -%}
{% set extends_list = [] -%}
{% if monitor -%}
{% do extends_list.append(template_monitor) -%}
{% endif -%}
{% do extends_list.append(logrotate_cfg) -%}
#############################
#
# Instanciate kvm
......@@ -18,6 +27,25 @@ parts =
cron
# cron-entry-monitor
frontend-promise
{% if monitor -%}
# monitor parts
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-status-history-cgi
deploy-status-cgi
# deploy-logfile-cgi
# deploy-resource-consumption-monitoring-cgi
setup-static-files
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
monitor-access-log
monitor-access-public
# monitor-frontend-promise
{% endif -%}
{% if slapparameter_dict.get('document-host', '') %}
cluster-url-path
{% endif -%}
......@@ -28,14 +56,21 @@ parts =
{% if slapparameter_dict.get('data-to-vm', '') %}
vm-data-content
{% endif -%}
{% if use_tap == 'true' and tap_network_dict.has_key('ipv4') %}
ansible-vm-promise
logrotate-vm-bootstrap
{% endif -%}
{% if slapparameter_dict.get('authorized-key', '') %}
get-authorized-key
{% endif -%}
# extends =
# {{ template_httpd_cfg }}
{% endif -%}
extends =
# Add extends list
{{ extends_list | join('\n ') }}
# {{ template_httpd_cfg }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
......@@ -289,6 +324,43 @@ hostname = ${httpd:host}
port = ${httpd:port}
{% endif %}
{% if monitor == 'true' -%}
[monitor-access-log]
< = monitor-directory-access
source = ${directory:log}
[monitor-access-public]
< = monitor-directory-access
source = ${directory:public}
[monitor-parameters]
port = 8026
{% if slapparameter_dict.get('document-host', '') and slapparameter_dict.get('document-port', '') -%}
# XXX - Set frontend software type to 'custom-personal' by default for cluster instance
{% set frontend_software_type = 'custom-personal' -%}
{% endif -%}
[request-monitor-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Monitor {{ slapparameter_dict.get('name', '') }} Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${monitor-parameters:url}
software-type = {{ slapparameter_dict.get('monitor-frontend-software-type', frontend_software_type) }}
return = site_url domain
[monitor-frontend-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/monitor_frontend
url = ${publish-connection-information:monitor_url}
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
check-secure = 1
{% endif -%}
[publish-connection-information]
recipe = slapos.cookbook:publish
ipv6 = ${slap-network-information:global-ipv6}
......@@ -323,6 +395,10 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
7_info = Get the publick key file in your VM with the command: wget {{ kvm_http }}/authorized_keys
{% endif %}
{% endif %}
{% if monitor -%}
monitor_url = ${request-monitor-frontend:connection-site_url}
monitor_v6_url = ${monitor-parameters:url}
{% endif -%}
{% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
1_info = Use these configurations below to configure interface {{ iface }} in your VM.
......@@ -378,16 +454,31 @@ recipe = plone.recipe.command
name = {{ slapparameter_dict.get('name', 'localhost') }}
{% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
local-ipv4 = ${slap-network-information:tap-ipv4}
gateway = ${slap-network-information:tap-gateway}
netmask = ${slap-network-information:tap-network}
network = ${slap-network-information:tap-netmask}
{% else -%}
local-ipv4 = 127.0.0.1
gateway =
netmask =
network =
{% endif -%}
path-host = ${directory:public}/hostname
path-ip = ${directory:public}/ipv4
path-gateway = ${directory:public}/gateway
path-network = ${directory:public}/network
path-netmask = ${directory:public}/netmask
command =
rm -f ${:path-host}
rm -f ${:path-ip}
rm -f ${:path-gateway}
rm -f ${:path-network}
rm -f ${:path-netmask}
echo "${:name}" > ${:path-host}
echo "${:local-ipv4}" > ${:path-ip}
echo "${:gateway}" > ${:path-gateway}
echo "${:network}" > ${:path-network}
echo "${:netmask}" > ${:path-netmask}
update-command = ${:command}
# To access documents of main instance (in case of kvm-cluster) through http
......@@ -412,6 +503,27 @@ shell-path = {{ dash_executable_location }}
6tunnel-path = {{ sixtunnel_executable_location }}
runner-path = ${directory:services}/6tunnel-cluster
[ansible-vm-promise]
recipe = slapos.recipe.template:jinja2
template = {{ ansible_promise_tpl }}
rendered = ${directory:promises}/ansible_{{ name }}
extensions = jinja2.ext.do
context =
key host slap-network-information:tap-ipv4
raw logs ${directory:public}/ansible
raw name {{ name }}
[logrotate-vm-bootstrap]
< = logrotate-entry-base
name = vm-bootstrap
log = ${directory:public}/ansible/vm-bootstrap.log
[logrotate-entry-base]
recipe = slapos.cookbook:logrotate.d
logrotate-entries = ${logrotate:logrotate-entries}
backup = ${logrotate:backup}
[slap-parameter]
# Default values if not specified
frontend-software-type = frontend
......
[buildout]
parts =
cron-entry-logrotate
[cron]
recipe = slapos.cookbook:cron
cron-entries = ${logrotate-directory:cron-entries}
dcrond-binary = {{ dcron_location }}/sbin/crond
crontabs = ${logrotate-directory:crontabs}
cronstamps = ${logrotate-directory:cronstamps}
catcher = ${cron-simplelogger:wrapper}
binary = ${logrotate-directory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = ${logrotate-directory:bin}/cron_simplelogger
log = ${logrotate-directory:log}/cron.log
[logrotate]
recipe = slapos.cookbook:logrotate
logrotate-entries = ${logrotate-directory:logrotate-entries}
backup = ${logrotate-directory:logrotate-backup}
logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
gzip-binary = {{ gzip_location }}/bin/gzip
gunzip-binary = {{ gzip_location }}/bin/gunzip
wrapper = ${logrotate-directory:bin}/logrotate
conf = ${logrotate-directory:etc}/logrotate.conf
state-file = ${logrotate-directory:srv}/logrotate.status
[cron-entry-logrotate]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = logrotate
frequency = 0 0 * * *
command = ${logrotate:wrapper}
[logrotate-directory]
recipe = slapos.cookbook:mkdirectory
cron-entries = ${:etc}/cron.d
cronstamps = ${:etc}/cronstamps
crontabs = ${:etc}/crontabs
logrotate-backup = ${:backup}/logrotate
logrotate-entries = ${:etc}/logrotate.d
bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
backup = ${:srv}/backup
etc = ${buildout:directory}/etc
services = ${:etc}/service
log = ${buildout:directory}/var/log
......@@ -66,8 +66,10 @@ template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
filename = template-kvm-cluster.cfg
extra-context =
section parameter_dict dynamic-template-kvm-cluster-parameters
raw logrotate_cfg ${template-logrotate-base:rendered}
raw template_content ${template-content:location}/${template-content:filename}
raw template_httpd_cfg ${template-httpd:rendered}
raw template_monitor ${monitor-template:output}
[dynamic-template-kvm]
recipe = slapos.recipe.template:jinja2
......@@ -81,10 +83,12 @@ context =
key slapparameter_dict slap-configuration:configuration
key storage_dict slap-configuration:storage-dict
key tap_network_dict slap-configuration:tap-network-information-dict
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw logrotate_cfg ${template-logrotate-base:rendered}
raw novnc_location ${noVNC:location}
raw netcat_bin ${netcat:location}/bin/netcat
raw openssl_executable_location ${openssl:location}/bin/openssl
......@@ -93,6 +97,7 @@ context =
raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
raw template_httpd_cfg ${template-httpd:rendered}
raw template_content ${template-content:location}/${template-content:filename}
raw template_monitor ${monitor-template:output}
raw websockify_executable_location ${buildout:directory}/bin/websockify
template-parts-destination = ${template-parts:destination}
template-replicated-destination = ${template-replicated:destination}
......
......@@ -16,6 +16,8 @@ pycrypto = 2.6.1
slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.7
smmap = 0.9.0
erp5.util = 0.4.42
pycurl = 7.19.5.1
# Required by:
# slapos.toolbox==0.48
......
#!/usr/bin/env python
# Parse Ansible result log file and define if execution succeed or not
import os
import json
FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout', 'stderr',
'response', 'status_code', 'url', 'dest']
ansible_log_dir = '{{ logs }}'
vm_name = '{{ name }}'
result_OK = '127.0.0.1_OK'
result_failed = '127.0.0.1_FAILED'
result_failed_ignore = '127.0.0.1_FAILED_IGNORED'
def get_log(res):
log = ""
if type(res) == type(dict()):
log = '>> Running task: %s, args [%s]\n' % (res['invocation']['module_name'],
res['invocation']['module_args'])
for field in FIELDS:
if field in res.keys():
# use default encoding, check out sys.setdefaultencoding
log += '\n{0}:\n{1}'.format(field, res[field])
return log
def file_to_dict(filepath):
content = '{}'
with open(filepath, 'r') as f:
content = f.read()
return json.loads(content)
def check_result():
success_file = os.path.join(ansible_log_dir, result_OK)
error_file = os.path.join(ansible_log_dir, result_failed)
if not len(os.listdir(ansible_log_dir)):
return (0, 'No Ansible promise uploaded!')
if os.path.exists(error_file):
result = file_to_dict(error_file)
if len(result) > 0:
#there is more that one failed task in the report
message = ''
for res in result:
message += '%s\n\n' % get_log(res)
return (0, message)
elif not os.path.exists(success_file):
return (0, 'All Ansible tasks failed to run')
return (1, '')
if __name__ == "__main__":
result = check_result()
if not result[0]:
raise Exception('Failed to run Ansible in %s, result is: \n%s' % (vm_name,
result[1]))
ServerLimit 2
ServerLimit 16
StartServers 1
MaxClients 2
MaxClients 7
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment