Merge branch 'master' into 1.0

component/dropbear/buildout.cfg

@@ -20,9 +20,11 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
 #      in order have all patches working.
 url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
 
+# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
+#      faster network throughput compared to dropbear defaults.
 configure-options =
   --with-zlib=${zlib:location}
-  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
+  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING -DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288"
 
 environment =
   CPPFLAGS =-I${zlib:location}/include

slapos/recipe/dropbear.py

@@ -182,15 +182,12 @@ def keysplit(s):
 class AddAuthorizedKey(GenericBaseRecipe):
 
   def install(self):
-    path_list = []
-
     ssh = self.createDirectory(self.options['home'], '.ssh')
-    path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
     for key in keysplit(self.options['key']):
       # XXX key might actually be the string 'None' or 'null'
       authorized_keys.append(key)
 
-    return path_list
+    return [authorized_keys.filename]
 

slapos/recipe/librecipe/generic.py

@@ -134,23 +134,25 @@ class GenericBaseRecipe(object):
       pidfile=None
   ):
     """
-    Creates a very simple (one command) shell script for process replacement.
+    Creates a shell script for process replacement.
     Takes care of quoting.
+    Takes care of #! line limitation when the wrapped command is a script.
     if pidfile parameter is specified, then it will make the wrapper a singleton,
     accepting to run only if no other instance is running.
     """
 
     lines = [ '#!/bin/sh' ]
 
-    for comment in comments:
-      lines.append('# %s' % comment)
+    if comments:
+      lines += '# ', '\n# '.join(comments), '\n'
 
-    if environment:
-      for key in environment:
+    lines.append('COMMAND=' + shlex.quote(command))
+
+    for key in environment or ():
       lines.append('export %s=%s' % (key, environment[key]))
 
     if pidfile:
-      lines.append(dedent("""\
+      lines.append(dedent("""
           # Check for other instances
           pidfile=%s
           if [ -e $pidfile ]; then
@@ -162,24 +164,31 @@ class GenericBaseRecipe(object):
               rm $pidfile
             fi
           fi
-          echo $$ > $pidfile""" % (pidfile, command)))
+          echo $$ > $pidfile""" % (shlex.quote(pidfile), command)))
 
-    lines.append('exec %s' % shlex.quote(command))
+    lines.append(dedent('''
+    # If the wrapped command uses a shebang, execute the referenced
+    # executable passing the script path as first argument.
+    # This is to workaround the limitation of 127 characters in #!
+    [ ! -f "$COMMAND" ] || {
+      [ "`head -c2`" != "#!" ] || read -r EXE ARG
+    } < "$COMMAND"
 
-    for param in parameters:
-      if len(lines[-1]) < 40:
-        lines[-1] += ' ' + shlex.quote(param)
-      else:
-        lines[-1] += ' \\'
-        lines.append('\t' + shlex.quote(param))
+    exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
 
+    parameters = map(shlex.quote, parameters)
     if parameters_extra:
       # pass-through further parameters
+      parameters.append('"$@"')
+    for param in parameters:
+      if len(lines[-1]) < 40:
+        lines[-1] += ' ' + param
+      else:
         lines[-1] += ' \\'
-        lines.append('\t"$@"')
+        lines.append('\t' + param)
 
-    content = '\n'.join(lines) + '\n'
-    return self.createFile(name, content, 0700)
+    lines.append('')
+    return self.createFile(name, '\n'.join(lines), 0700)
 
   def createDirectory(self, parent, name, mode=0700):
     path = os.path.join(parent, name)

slapos/recipe/wrapper.py

@@ -54,8 +54,16 @@ class Recipe(GenericBaseRecipe):
         if environment is not None:
             environment = dict((k.strip(), v.strip()) for k, v in [
               line.split('=') for line in environment.splitlines() if line.strip() ])
-        return [self.createPythonScript(
-            wrapper_path,
+
+        # We create a python script and a wrapper around the python
+        # script because the python script might have a too long #! line
+        python_script = self.createPythonScript(
+            wrapper_path+'.py',
             'slapos.recipe.librecipe.execute.generic_exec',
-            (command_line, wait_files, environment,),
-        )]
+            (command_line, wait_files, environment,), )
+        return [python_script, self.createWrapper(
+             name=wrapper_path,
+             command=python_script,
+             parameters=[],
+             parameters_extra=parameters_extra) ]
+

software/apache-frontend/common.cfg

@@ -58,27 +58,27 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 16b22ef9b0476352e2e5b68f5966ce62
+md5sum = 51752d0e4eae8c761750117c44983594 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = 00361d1e4a2788863de58e0ca9b6dfcf
+md5sum = 07b6f864b46cb2f7b4b8aab71da95d13 
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = c495395bf4cae93ff665c5e74e6d4583 
+md5sum = a05bc0f925902ddf0b4d12146b10c9e1 
 mode = 0644
 
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = f09759a0b68337d820c70e72afb4dbf9
+md5sum = c01c0ebb749d16bda4649bf42d36c7ea 
 mode = 640
 
 [template-slave-configuration]

software/apache-frontend/instance-apache-frontend.cfg

@@ -33,28 +33,9 @@ parts =
   trafficserver-storage-config
   trafficserver-promise-listen-port
 
-## Monitoring part
-###Parts to add for monitoring
-  certificate-authority
-  cron-entry-monitor
-  cron-entry-rss
-  deploy-index
-  deploy-settings-cgi
-  deploy-status-cgi
-  deploy-status-history-cgi
-  setup-static-files
-  certificate-authority
-  zero-parameters
-  public-symlink
-  cgi-httpd-wrapper
-  cgi-httpd-graceful-wrapper
-  monitor-promise
-  monitor-instance-log-access
 ## Monitor for apache
-  monitor-current-log-access
-  monitor-backup-log-access
+  monitor-base
   monitor-ats-cache-stats-wrapper
-  monitor-apache-configuration-verification
   monitor-verify-re6st-connectivity
 
 extends = ${monitor-template:output}
@@ -74,9 +55,9 @@ srv = $${buildout:directory}/srv/
 var = $${buildout:directory}/var/
 template = $${buildout:directory}/template/
 
-backup = $${:srv}/backup
-log = $${:var}/log
-run = $${:var}/run
+backup = $${buildout:directory}/srv/backup
+log = $${buildout:directory}/var/log
+run = $${buildout:directory}/var/run
 service = $${:etc}/service
 etc-run = $${:etc}/run
 promise = $${:etc}/promise
@@ -181,7 +162,7 @@ extra-context =
     section logrotate_dict logrotate
     section frontend_configuration frontend-configuration
     section apache_configuration apache-configuration
-    key monitor_url monitor-parameters:url
+    key monitor_base_url monitor-instance-parameter:monitor-base-url 
 
 [dynamic-custom-group-template-slave-list]
 < = jinja2-template-base
@@ -633,19 +614,15 @@ frontend-name =
 # Monitoring sections
 #
 
-[monitor-current-log-access]
-< = monitor-directory-access
-source = $${directory:log}
-
-[monitor-backup-log-access]
-< = monitor-directory-access
-source = $${directory:logrotate-backup}
+[monitor-conf-parameters]
+private-path-list += 
+  $${directory:logrotate-backup}
 
 # Produce ATS Cache stats
 [monitor-ats-cache-stats-wrapper]
 < = jinja2-template-base
 template = ${template-wrapper:output}
-rendered = $${monitor-directory:monitoring-cgi}/ats-cache-stats
+rendered = $${monitor-directory:report}/ats-cache-stats
 mode = 0700
 command = export TS_ROOT=$${buildout:directory} && echo "<pre>$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})</pre>"
 extra-context =
@@ -659,19 +636,9 @@ mode = 644
 context =
     raw content show:cache-stats
 
-# Display result of apache configuration check
-[monitor-apache-configuration-verification]
-< = jinja2-template-base
-template = ${template-wrapper:output}
-rendered = $${monitor-directory:monitoring-cgi}/front-httpd-configuration
-mode = 0700
-command = echo "<pre>$($${apache-configuration:frontend-configuration-verification})</pre>"
-extra-context =
-  key content :command
-
 [monitor-verify-re6st-connectivity]
 recipe = slapos.cookbook:check_url_available
-path = $${monitor-directory:monitor-custom-scripts}/re6st-connectivity
+path = $${directory:promise}/re6st-connectivity
 url = $${public:re6st-verification-url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -80,6 +80,10 @@ context =
 [replicate]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
+config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+config-monitor-username = ${monitor-htpasswd:username}
+config-monitor-password = ${monitor-htpasswd:passwd}
+
 {% set frontend_software_url_key = "-frontend-software-release-url" %}
 {% if slapparameter_dict.has_key(frontend_software_url_key) %}
 software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
@@ -87,8 +91,7 @@ software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
 software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{frontend_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-connection-monitor_url =
+return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
 
 {% for section, frontend_request in request_dict.iteritems() %}
 [{{section}}]
@@ -117,13 +120,10 @@ slave-amount = {{ slave_instance_list | length }}
 accepted-slave-amount = {{ authorized_slave_list | length }}
 rejected-slave-amount = {{ rejected_slave_list | length }}
 rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
-{% for frontend in frontend_section_list %}
-{{ frontend }}-monitor-url = {{ '${' + frontend + ':connection-monitor_url}' }}
-{% endfor -%}
-{% for frontend in frontend_list -%}
-#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
-{% endfor -%}
-
+monitor-base-url = ${monitor-conf-parameters:base-url}
+monitor-url = ${:monitor-base-url}/public/feeds
+monitor-user = ${monitor-instance-parameter:username}
+monitor-password = ${monitor-instance-parameter:password}
 
 #----------------------------
 #--
@@ -149,8 +149,16 @@ extensions = jinja2.ext.do
 extra-context =
     section slave_information slave-information
 
+[monitor-conf-parameters]
+monitor-url-list +=
+{% for frontend in frontend_section_list %}
+{{ '  ${' + frontend + ':connection-monitor-base-url}' }}
+{% endfor -%}
+
 [buildout]
+extends = {{ template_monitor }}
 parts =
+  monitor-base
   publish-slave-information
   publish-information
 {% for part in part_list -%}

software/apache-frontend/instance.cfg

@@ -48,3 +48,5 @@ extra-context =
     raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default-custom-personal-custom-group-replicate
+    raw template_monitor ${monitor2-template:rendered}
+

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -262,7 +262,7 @@ private-ipv4 = {{ local_ipv4 }}
 {% if extra_slave_instance_list -%}
 slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
 {% endif -%}
-monitor_url = {{ monitor_url }}
+monitor-base-url = {{ monitor_base_url }}
 
 [buildout]
 parts +=

software/kvm/instance-kvm-resilient-input-schema.json

@@ -35,14 +35,14 @@
                     "title": "Remove backups older than...",
                     "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                     "type": "string",
-                    "default": "3B",
+                    "default": "2W",
                     "optional": true
                 },
                 "resilient-clone-number": {
                     "title": "Amount of backup(s) to create",
                     "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                     "type": "integer",
-                    "default": 2,
+                    "default": 1,
                     "optional": true
                 },
                 "ignore-known-hosts-file": {

software/slaprunner/instance-runner-input-schema.json

@@ -50,6 +50,17 @@
             "description": "Software type of your instance inside the runner",
             "type": "string"
         },
+        "cpu-usage-ratio": {
+            "title": "CPU Usage Ratio",
+            "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
+            "type": "integer",
+            "default" : 4
+        },
+        "instance-name": {
+            "title": "Displayed instance name",
+            "description": "Name of the instance, to show in the window title",
+            "type": "string"
+        },
         "custom-frontend-backend-url": {
             "title": "Custom Frontend Backend URL",
             "description": "return an ipv4 frontend of the given ipv6(+optional port)",
@@ -110,22 +121,6 @@
             "description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
             "type": "string",
             "default": "monitor.app.officejs.com"
-        },
-        "cpu-usage-ratio": {
-            "title": "CPU Usage Ratio",
-            "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
-            "type": "integer",
-            "default" : 4
-        },
-        "resilient-clone-number": {
-            "title": "Number of Resilient Clones",
-            "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
-            "type": "integer",
-            "enum": [1, 2]
-        },
-        "instance-name": {
-            "description": "Name of the instance, to show in the window title",
-            "type": "string"
         }
     }
 }

software/slaprunner/instance-runner-resilient-input-schema.json

@@ -35,14 +35,15 @@
                     "title": "Remove backups older than...",
                     "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                     "type": "string",
-                    "default": "3B",
+                    "default": "2W",
                     "optional": true
                 },
                 "resilient-clone-number": {
                     "title": "Amount of backup(s) to create",
                     "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                     "type": "integer",
-                    "default": 2,
+                    "default": 1,
+                    "enum": [1, 2],
                     "optional": true
                 },
                 "ignore-known-hosts-file": {

stack/monitor/buildout.cfg

@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = eab45fd2b0f017ab97a9c8f466c8dceb
+md5sum = cd7f386fe1b5066d8508758249b408d3  
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -130,7 +130,7 @@ depends =
 [monitor2-bin]
 <= monitor-template-script
 filename = monitor.py
-md5sum = 222365a469f8ab08a0367d81c0b03982
+md5sum = 31beec15d3c3cd7979d04ecb834c439a
 
 [run-promise-py]
 recipe = slapos.recipe.template:jinja2

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -61,6 +61,7 @@ services = ${:etc}/service
 promises = ${:etc}/promise
 monitor = ${:srv}/monitor
 monitor-promise = ${:etc}/monitor-promise
+monitor-report = ${:etc}/monitor-report
 
 [monitor-directory]
 recipe = slapos.cookbook:mkdirectory
@@ -69,6 +70,7 @@ etc = ${directory:etc}
 run = ${directory:monitor}/run
 #run = ${directory:scripts}
 promises = ${directory:monitor-promise}
+report = ${directory:monitor-report}
 pids = ${directory:run}/monitor
 cgi-bin = ${directory:monitor}/cgi-bin
 webdav = ${directory:monitor}/webdav
@@ -374,7 +376,7 @@ monitor-httpd-port = 8196
 monitor-base-url = ${monitor-frontend-promise:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 opml-url-list =
-cors-domains = 
+cors-domains = monitor.app.officejs.com 
 # XXX Hard coded parameter
 collector-db = /srv/slapgrid/var/data-log/collector.db
 # Credentials

stack/monitor/scripts/monitor.py

@@ -222,13 +222,18 @@ class Monitoring(object):
   def getMonitorTitleFromUrl(self, monitor_url):
     # This file should be generated
     if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'):
-      return 'Unknow Instance'
+      return 'Unknown Instance'
     if not monitor_url.endswith('/'):
       monitor_url = monitor_url + '/'
-    context = ssl._create_unverified_context()
+
     url  = monitor_url + '/.jio_documents/monitor.global.json' # XXX Hard Coded path
     try:
+      # XXX - working here with public url
+      if hasattr(ssl, '_create_unverified_context'):
+        context = ssl._create_unverified_context()
         response = urllib2.urlopen(url, context=context)
+      else:
+        response = urllib2.urlopen(url)
     except urllib2.HTTPError:
       return 'Unknow Instance'
     else: