Merge branch 'master' into 1.0

component/pycurl/buildout.cfg

+[buildout]
+extends =
+  ../curl/buildout.cfg
+  ../openssl/buildout.cfg
+
+parts =
+  pycurl
+
+[pycurl-env]
+PATH = ${curl:location}/bin:${openssl:location}/bin:%(PATH)s
+PYCURL_SSL_LIBRARY=openssl
+CPPFLAGS=-I${openssl:location}/include
+CFLAGS=-I${openssl:location}/include
+
+[pycurl]
+recipe = zc.recipe.egg:custom
+egg = pycurl
+
+rpath =
+  ${curl:location}/lib/
+  ${openssl:location}/lib/
+environment = pycurl-env

software/kvm/common.cfg

@@ -12,9 +12,11 @@ extends =
   ../../component/openssl/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/netcat/buildout.cfg
+  ../../component/pycurl/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../stack/resilient/buildout.cfg
+  ../../stack/monitor/buildout.cfg
 
 # stacks are listed from most generic to most specific,
 # to avoid versioning issues
@@ -51,6 +53,7 @@ eggs =
   cns.recipe.symlink
   collective.recipe.template
   plone.recipe.command
+  ${pycurl:egg}
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -87,7 +90,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 3bca2c959d19881270c64f94ad1ebba8
+md5sum = 41eb12b66bd1af86b1b98b2166e86417
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -95,7 +98,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = ea1e8f4a7c1878beec83267fd40728c2
+md5sum = da3b3bde94ac2c06759a89c78168df91
 download-only = true
 on-update = true
 
@@ -103,7 +106,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 5a864099760e3a37fa4604044d708657
+md5sum = 6ac9d1b06398b57cc1c8d63a9c492415
 download-only = true
 on-update = true
 
@@ -168,12 +171,20 @@ md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
 
+[template-ansible-promise]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/ansible-promise.in
+md5sum = d0bd07d5a7799b7aea7720ffdf46f322
+mode = 0644
+download-only = true
+filename = ansible-promise.in
+
 [template-apache-conf]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/apache.conf.in
 mode = 644
 filename = apache.conf.in
-md5sum = e9c9fd88d71e9dc7416149af5bcfb951
+md5sum = 355fdabdb86fee8e9714b6d357149958
 download-only = true
 on-update = true
 
@@ -186,6 +197,17 @@ md5sum = 822737e483864bf255ad1259237bef2a
 download-only = true
 on-update = true
 
+[template-logrotate-base]
+recipe = slapos.recipe.template:jinja2
+filename = instance-logrotate-base.cfg
+template = ${:_profile_base_location_}/instance-logrotate-base.cfg.in
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-logrotate-base.cfg
+md5sum = f28fbd310944f321ccb34b2a34c82005
+context =
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+
 [template-httpd]
 recipe = slapos.recipe.template:jinja2
 filename = template-httpd.cfg

software/kvm/instance-kvm-cluster-input-schema.json

@@ -294,6 +294,12 @@
               "title": "Text content to send to this virtual machine.",
               "description": "Text content which will be written in a file 'data' of http server of this virtual machine instance. The file will be available via URL: http://10.0.2.100/data in the VM.",
               "type": "string"
+            },
+            "enable-monitor": {
+              "title": "Deploy monitoring tools",
+              "description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
+              "type": "boolean",
+              "default": true
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -64,6 +64,7 @@ config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
 {% if kvm_parameter_dict.get('data-to-vm', '') -%}
 config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
 {% endif -%}
+config-enable-monitor = True
 
 # Enable simple http server on ipv6 so all VMs will access it
 config-document-host = ${apache-conf:ip}

software/kvm/instance-kvm-input-schema.json

@@ -135,6 +135,12 @@
       "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
       "type": "string"
     },
+    "enable-monitor": {
+      "title": "Deploy monitoring tools",
+      "description": "Deploy monitor instance to this kvm instance. It help to check instance status, log and promise results.",
+      "type": "boolean",
+      "default": true
+    },
     "enable-http-server": {
       "title": "Enable local http server",
       "description": "Set if local http server which serve files to the vm should be deployed. If set to true, get file into the vm with URL: http://10.0.2.100/FILE.",

software/kvm/instance-kvm.cfg.jinja2

 {% set enable_http = slapparameter_dict.get('enable-http-server', 'False').lower() -%}
 {% set use_tap = slapparameter_dict.get('use-tap', 'False').lower() -%}
 {% set use_nat = slapparameter_dict.get('use-nat', 'True').lower() -%}
+{% set name = slapparameter_dict.get('name', 'localhost') -%}
+{% set monitor = slapparameter_dict.get('enable-monitor', 'True').lower()  -%}
+{% set frontend_software_type = 'default' -%}
+{% set extends_list = [] -%}
+
+{% if monitor -%}
+{% do extends_list.append(template_monitor) -%}
+{% endif -%}
+{% do extends_list.append(logrotate_cfg) -%}
 #############################
 #
 # Instanciate kvm
@@ -18,6 +27,25 @@ parts =
   cron
 #  cron-entry-monitor
   frontend-promise
+{% if monitor -%}
+# monitor parts
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-status-history-cgi
+  deploy-status-cgi
+#  deploy-logfile-cgi
+#  deploy-resource-consumption-monitoring-cgi
+  setup-static-files
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+  monitor-access-log
+  monitor-access-public
+#  monitor-frontend-promise
+{% endif -%}
 {% if slapparameter_dict.get('document-host', '') %}
   cluster-url-path
 {% endif -%}
@@ -28,14 +56,21 @@ parts =
 {% if slapparameter_dict.get('data-to-vm', '') %}
   vm-data-content
 {% endif -%}
+{% if use_tap == 'true' and tap_network_dict.has_key('ipv4') %}
+  ansible-vm-promise
+  logrotate-vm-bootstrap
+{% endif -%}
 {% if slapparameter_dict.get('authorized-key', '') %}
   get-authorized-key
 {%   endif -%}
-  
-# extends = 
-#  {{ template_httpd_cfg }}
+
 {% endif -%}
 
+extends = 
+#  Add extends list
+ {{ extends_list | join('\n  ') }}
+#  {{ template_httpd_cfg }}
+
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
@@ -289,6 +324,43 @@ hostname = ${httpd:host}
 port = ${httpd:port}
 {% endif %}
 
+{% if monitor == 'true' -%}
+[monitor-access-log]
+< = monitor-directory-access
+source = ${directory:log}
+
+[monitor-access-public]
+< = monitor-directory-access
+source = ${directory:public}
+
+[monitor-parameters]
+port = 8026
+
+{% if slapparameter_dict.get('document-host', '') and slapparameter_dict.get('document-port', '') -%}
+# XXX - Set frontend software type to 'custom-personal' by default for cluster instance
+{% set frontend_software_type = 'custom-personal' -%}
+{% endif -%}
+
+[request-monitor-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Monitor {{ slapparameter_dict.get('name', '') }} Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${monitor-parameters:url}
+software-type = {{ slapparameter_dict.get('monitor-frontend-software-type', frontend_software_type) }}
+return = site_url domain
+
+[monitor-frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/monitor_frontend
+url = ${publish-connection-information:monitor_url}
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+check-secure = 1
+{% endif -%}
+
 [publish-connection-information]
 recipe = slapos.cookbook:publish
 ipv6 = ${slap-network-information:global-ipv6}
@@ -323,6 +395,10 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
 7_info = Get the publick key file in your VM with the command: wget {{ kvm_http }}/authorized_keys
 {%   endif %}
 {% endif %}
+{% if monitor -%}
+monitor_url = ${request-monitor-frontend:connection-site_url}
+monitor_v6_url = ${monitor-parameters:url}
+{% endif -%}
 
 {% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
 1_info = Use these configurations below to configure interface {{ iface }} in your VM.
@@ -378,16 +454,31 @@ recipe = plone.recipe.command
 name = {{ slapparameter_dict.get('name', 'localhost') }}
 {% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
 local-ipv4 = ${slap-network-information:tap-ipv4}
+gateway = ${slap-network-information:tap-gateway}
+netmask = ${slap-network-information:tap-network}
+network = ${slap-network-information:tap-netmask}
 {% else -%}
 local-ipv4 = 127.0.0.1
+gateway =
+netmask =
+network =
 {% endif -%}
 path-host = ${directory:public}/hostname
 path-ip = ${directory:public}/ipv4
+path-gateway = ${directory:public}/gateway
+path-network = ${directory:public}/network
+path-netmask = ${directory:public}/netmask
 command = 
   rm -f ${:path-host}
   rm -f ${:path-ip}
+  rm -f ${:path-gateway}
+  rm -f ${:path-network}
+  rm -f ${:path-netmask}
   echo "${:name}" > ${:path-host}
   echo "${:local-ipv4}" > ${:path-ip}
+  echo "${:gateway}" > ${:path-gateway}
+  echo "${:network}" > ${:path-network}
+  echo "${:netmask}" > ${:path-netmask}
 update-command = ${:command}
 
 # To access documents of main instance (in case of kvm-cluster) through http
@@ -412,6 +503,27 @@ shell-path = {{ dash_executable_location }}
 6tunnel-path = {{ sixtunnel_executable_location }}
 runner-path = ${directory:services}/6tunnel-cluster
 
+[ansible-vm-promise]
+recipe = slapos.recipe.template:jinja2
+template = {{ ansible_promise_tpl }}
+rendered = ${directory:promises}/ansible_{{ name }}
+extensions = jinja2.ext.do
+context =
+  key host slap-network-information:tap-ipv4
+  raw logs ${directory:public}/ansible
+  raw name {{ name }}
+
+
+[logrotate-vm-bootstrap]
+< = logrotate-entry-base
+name = vm-bootstrap
+log = ${directory:public}/ansible/vm-bootstrap.log
+
+[logrotate-entry-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+
 [slap-parameter]
 # Default values if not specified
 frontend-software-type = frontend

software/kvm/instance-logrotate-base.cfg.in

+[buildout]
+parts =
+  cron-entry-logrotate
+
+[cron]
+recipe = slapos.cookbook:cron
+cron-entries = ${logrotate-directory:cron-entries}
+dcrond-binary = {{ dcron_location }}/sbin/crond
+crontabs = ${logrotate-directory:crontabs}
+cronstamps = ${logrotate-directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${logrotate-directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${logrotate-directory:bin}/cron_simplelogger
+log = ${logrotate-directory:log}/cron.log
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+logrotate-entries = ${logrotate-directory:logrotate-entries}
+backup = ${logrotate-directory:logrotate-backup}
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+wrapper = ${logrotate-directory:bin}/logrotate
+conf = ${logrotate-directory:etc}/logrotate.conf
+state-file = ${logrotate-directory:srv}/logrotate.status
+
+[cron-entry-logrotate]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate-directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+crontabs = ${:etc}/crontabs
+logrotate-backup = ${:backup}/logrotate
+logrotate-entries = ${:etc}/logrotate.d
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+backup = ${:srv}/backup
+etc = ${buildout:directory}/etc
+services = ${:etc}/service
+log = ${buildout:directory}/var/log
+

software/kvm/instance.cfg.in

@@ -66,8 +66,10 @@ template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
 filename = template-kvm-cluster.cfg
 extra-context =
   section parameter_dict dynamic-template-kvm-cluster-parameters
+  raw logrotate_cfg ${template-logrotate-base:rendered}
   raw template_content ${template-content:location}/${template-content:filename}
   raw template_httpd_cfg ${template-httpd:rendered}
+  raw template_monitor ${monitor-template:output}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
@@ -81,10 +83,12 @@ context =
     key slapparameter_dict slap-configuration:configuration
     key storage_dict slap-configuration:storage-dict
     key tap_network_dict slap-configuration:tap-network-information-dict
+    raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
     raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
+    raw logrotate_cfg ${template-logrotate-base:rendered}
     raw novnc_location ${noVNC:location}
     raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
@@ -93,6 +97,7 @@ context =
     raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
     raw template_httpd_cfg ${template-httpd:rendered}
     raw template_content ${template-content:location}/${template-content:filename}
+    raw template_monitor ${monitor-template:output}
     raw websockify_executable_location ${buildout:directory}/bin/websockify
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}

software/kvm/software.cfg

@@ -16,6 +16,8 @@ pycrypto = 2.6.1
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.7
 smmap = 0.9.0
+erp5.util = 0.4.42
+pycurl = 7.19.5.1
 
 # Required by:
 # slapos.toolbox==0.48

software/kvm/template/ansible-promise.in

+#!/usr/bin/env python
+
+# Parse Ansible result log file and define if execution succeed or not
+
+import os
+import json
+
+FIELDS = ['cmd', 'command', 'start', 'end', 'delta', 'msg', 'stdout', 'stderr',
+          'response', 'status_code', 'url', 'dest']
+
+ansible_log_dir = '{{ logs }}'
+vm_name = '{{ name }}'
+result_OK = '127.0.0.1_OK'
+result_failed = '127.0.0.1_FAILED'
+result_failed_ignore = '127.0.0.1_FAILED_IGNORED'
+
+def get_log(res):
+  log = ""
+  if type(res) == type(dict()):
+    log = '>> Running task: %s, args [%s]\n' % (res['invocation']['module_name'],
+                                      res['invocation']['module_args'])
+    for field in FIELDS:
+      if field in res.keys():
+      	# use default encoding, check out sys.setdefaultencoding
+        log += '\n{0}:\n{1}'.format(field, res[field])
+
+  return log
+
+def file_to_dict(filepath):
+  content = '{}'
+  with open(filepath, 'r') as f:
+    content = f.read()
+  return json.loads(content)
+
+def check_result():
+  success_file = os.path.join(ansible_log_dir, result_OK)
+  error_file = os.path.join(ansible_log_dir, result_failed)
+
+  if not len(os.listdir(ansible_log_dir)):
+    return (0, 'No Ansible promise uploaded!')
+  if os.path.exists(error_file):
+    result = file_to_dict(error_file)
+    if len(result) > 0:
+      #there is more that one failed task in the report
+      message = ''
+      for res in result:
+        message += '%s\n\n' % get_log(res)
+      return (0, message)
+  elif not os.path.exists(success_file):
+    return (0, 'All Ansible tasks failed to run')
+  return (1, '')
+
+if __name__ == "__main__":
+  result = check_result()
+  if not result[0]:
+    raise Exception('Failed to run Ansible in %s, result is: \n%s' % (vm_name,
+                          result[1]))

software/kvm/template/apache.conf.in

-ServerLimit 2
+ServerLimit 16
 StartServers 1
-MaxClients 2
+MaxClients 7
 
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so