Merge branch 'master' into slaprunner

slapos/recipe/apache_frontend/__init__.py

@@ -33,6 +33,7 @@ import zc.buildout
 import zc.recipe.egg
 import ConfigParser
 import re
+import traceback
 
 
 class Recipe(BaseSlapRecipe):
@@ -78,7 +79,7 @@ class Recipe(BaseSlapRecipe):
       # Future work may allow to choose between http and https (or both?)
       scheme = 'http://'
 
-      self.logger.info('processing slave instance: %s' % reference)
+      self.logger.info('Processing slave instance: %s' % reference)
 
       # Check for mandatory slave fields
       if backend_url is None:
@@ -88,8 +89,14 @@ class Recipe(BaseSlapRecipe):
 
       # Check for custom domain (like mypersonaldomain.com)
       # If no custom domain, use generated one.
-      domain = slave_instance.get('custom_domain', 
-          "%s.%s" % (reference.replace("-", "").lower(), frontend_domain_name))
+      # Note: if we get an empty custom_domain parameter, we ignore it
+      domain = slave_instance.get('custom_domain')
+      if isinstance(domain, basestring):
+        domain = domain.strip()
+      if domain is None or domain.strip() == '':
+        domain = "%s.%s" % (reference.replace("-", "").lower(),
+            frontend_domain_name)
+
       # Define the URL where the instance will be available
       # WARNING: we use default ports (443, 80) here.
       slave_dict[reference] = "%s%s/" % (scheme, domain)
@@ -149,7 +156,13 @@ class Recipe(BaseSlapRecipe):
 
     # Send connection informations about each slave
     for reference, url in slave_dict.iteritems():
-      self.setConnectionDict(dict(site_url=url), reference)
+      self.logger.debug("Sending connection parameters of slave "
+          "instance: %s" % reference)
+      try:
+        self.setConnectionDict(dict(site_url=url), reference)
+      except:
+        self.logger.fatal("Error while sending slave %s informations: %s",
+            reference, traceback.format_exc())
 
     # Then set it for master instance
     self.setConnectionDict(
@@ -451,7 +464,7 @@ class Recipe(BaseSlapRecipe):
     return stunnel_conf
 
   def installFrontendApache(self, ip_list, key, certificate, name,
-                            port=4443, plain_http_port=8080, 
+                            port=4443, plain_http_port=8080,
                             rewrite_rule_list=[], rewrite_rule_zope_list=[],
                             access_control_string=None):
     # Create htdocs, populate it with default 404 document

slapos/recipe/apachephp/__init__.py

@@ -85,28 +85,30 @@ class Recipe(GenericBaseRecipe):
       with open(secret_key_filename, 'r') as secret_key_file:
         secret_key = secret_key_file.read()
 
-    application_conf = dict(mysql_database=self.options['mysql-database'],
-                            mysql_user=self.options['mysql-username'],
-                            mysql_password=self.options['mysql-password'],
-                            mysql_host='%s:%s' % (self.options['mysql-host'],
-                                                  self.options['mysql-port']),
-                            secret_key=secret_key,
-                           )
+    # Generate application configuration file
+    if self.options.get('template'):
+      application_conf = dict(mysql_database=self.options['mysql-database'],
+                              mysql_user=self.options['mysql-username'],
+                              mysql_password=self.options['mysql-password'],
+                              mysql_host='%s:%s' % (self.options['mysql-host'],
+                                                    self.options['mysql-port']),
+                              secret_key=secret_key,
+                             )
 
-    directory, file_ = os.path.split(self.options['configuration'])
+      directory, file_ = os.path.split(self.options['configuration'])
 
-    path = self.options['htdocs']
-    if directory:
-      path = os.path.join(path, directory)
-      if not os.path.exists(path):
-        os.makedirs(path)
-      if not os.path.isdir(path):
-        raise OSError("Cannot create %r." % path)
+      path = self.options['htdocs']
+      if directory:
+        path = os.path.join(path, directory)
+        if not os.path.exists(path):
+          os.makedirs(path)
+        if not os.path.isdir(path):
+          raise OSError("Cannot create %r." % path)
 
-    destination = os.path.join(path, file_)
-    config = self.createFile(destination,
-      self.substituteTemplate(self.options['template'], application_conf))
-    path_list.append(config)
+      destination = os.path.join(path, file_)
+      config = self.createFile(destination,
+        self.substituteTemplate(self.options['template'], application_conf))
+      path_list.append(config)
 
     if os.path.exists(self.options['pid-file']):
       # Reload apache configuration

software/drupal/instance.cfg

-[buildout]
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-parts = instance
-
-[instance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-source = ${application:location}
-
-httpd_binary = ${apache:location}/bin/httpd
-mysql_binary = ${mariadb:location}/bin/mysql
-mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
-mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld_binary = ${mariadb:location}/libexec/mysqld

software/drupal/software.cfg

 [buildout]
-versions = versions
 
-parts =
-  template
-  apache-php
-  mariadb
-  eggs
-  instance-recipe-egg
-  downloadcache-workaround
-  
 extends =
-  ../../stack/lamp.cfg
-  ../../stack/shacache-client.cfg
+  ../../stack/lamp/buildout.cfg
 
 [application]
 recipe = slapos.recipe.build:download-unpacked
-url = http://ftp.drupal.org/files/projects/drupal-7.8.tar.gz
-md5sum = e0226b56e8d5c57c6b126e8ed5866b1f
-  
-[instance-recipe]
-egg = slapos.cookbook
-module = lamp.simple
-
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-#md5sum = Student shall put md5 of instance.cfg here
-output = ${buildout:directory}/template.cfg
+url = http://ftp.drupal.org/files/projects/drupal-7.14.tar.gz
+md5sum = af7abd95c03ecad4e1567ed94a438334
+
+[application-template]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/settings.php.in
+md5sum = add9e4e96094a9d3fb7cf0f4d26ae544
+download-only = True
+filename = template.in
 mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[versions]
-# Use SlapOS patched zc.buildout
-zc.buildout = 1.5.3-dev-SlapOS-010
-
-Jinja2 = 2.6
-MySQL-python = 1.2.3
-Werkzeug = 0.8.1
-apache-libcloud = 0.5.2
-buildout-versions = 1.7
-hexagonit.recipe.cmmi = 1.5.0
-meld3 = 0.6.7
-plone.recipe.command = 1.1
-slapos.cookbook = 0.34
-slapos.recipe.build = 0.7
-slapos.recipe.download = 1.0.dev-r4053
-slapos.recipe.template = 2.2
-slapos.toolbox = 0.10
-
-# Required by:
-# slapos.core==0.18
-# slapos.toolbox==0.10
-Flask = 0.8
-
-# Required by:
-# slapos.cookbook==0.34
-PyXML = 0.8.4
-
-# Required by:
-# hexagonit.recipe.cmmi==1.5.0
-hexagonit.recipe.download = 1.5.0
-
-# Required by:
-# slapos.cookbook==0.34
-# slapos.core==0.18
-# slapos.toolbox==0.10
-# xml-marshaller==0.9.7
-lxml = 2.3.1
-
-# Required by:
-# slapos.cookbook==0.34
-netaddr = 0.7.6
-
-# Required by:
-# slapos.core==0.18
-netifaces = 0.5
-
-# Required by:
-# slapos.toolbox==0.10
-paramiko = 1.7.7.1
-
-# Required by:
-# slapos.toolbox==0.10
-psutil = 0.3.0
-
-# Required by:
-# slapos.cookbook==0.34
-# slapos.core==0.18
-# slapos.toolbox==0.10
-setuptools = 0.6c12dev-r88846
-
-# Required by:
-# slapos.cookbook==0.34
-# slapos.toolbox==0.10
-slapos.core = 0.18
-
-# Required by:
-# slapos.core==0.18
-supervisor = 3.0a10
-
-# Required by:
-# slapos.cookbook==0.34
-# slapos.toolbox==0.10
-xml-marshaller = 0.9.7
-
-# Required by:
-# slapos.cookbook==0.34
-zc.recipe.egg = 1.3.2
-
-# Required by:
-# slapos.core==0.18
-zope.interface = 3.8.0
-
-[downloadcache-workaround]
-# workaround irritating problem of hexagonit.recipe.cmmi which automatically
-# creates download cache, which in turn switches builout to "semi-offline" mode
-recipe = plone.recipe.command
-# in hexagonit.recipe.cmmi if there is no ${buildout:download-cache} set it resolves
-# to ${buildout:directory}/downloads but this variable is available late, that's
-# why it is hardcoded only for required case
-download-cache = ${buildout:directory}/downloads
-command = [ -d ${:download-cache} ] && rm -fr ${:download-cache}/* || exit 0
-update-command = ${:command}
-stop-on-error = True
+[application-configuration]
+location = default.settings.php

stack/lamp.cfg

@@ -20,9 +20,6 @@ extends =
   ../component/pycrypto-python/buildout.cfg
   ../component/mysql-python/buildout.cfg
   ../stack/slapos.cfg
-  ../component/python-2.7/buildout.cfg
-# python-2.7 component is here only for compatibility with old software.cfg.
-# It is not needed and should not be used in LAMP-based software.cfg
 
 [eggs]
 recipe = zc.recipe.egg

stack/lamp/buildout.cfg

@@ -67,7 +67,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-php.cfg
 output = ${buildout:directory}/template-apache-php.cfg
-md5sum = f8a1599f19a67d948f4a1f32f3bd3e2f
+md5sum = 286b60a95d5ed6318fbdd92f834b7339
 mode = 0644
 
 [template-apache-backup]
@@ -119,6 +119,14 @@ output = ${buildout:directory}/template-pull-backup.cfg
 md5sum = 9aab30ba5aa23a37d4b507e7c414be00
 mode = 0644
 
+# Dummy parts in case no application configuration file is needed
+[application-template]
+filename =
+location =
+
+[application-configuration]
+location =
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =

stack/lamp/instance-apache-php.cfg

@@ -16,64 +16,36 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[promise]
-recipe = slapos.cookbook:check_port_listening
-path = $${basedirectory:promises}/apache
-hostname = $${apache-php:ip}
-port = $${apache-php:port}
-
-[frontend-promise]
-recipe = slapos.cookbook:check_url_available
-path = $${basedirectory:promises}/frontend
-url = $${request-frontend:connection-site_url}
-dash_path = ${dash:location}/bin/dash
-
-[publish-connection-informations]
-recipe = slapos.cookbook:publish
-backend_url = $${apache-php:url}
-url = $${request-frontend:connection-site_url}
-
-[mariadb-urlparse]
-recipe = slapos.cookbook:urlparse
-url = $${request-mariadb:connection-url}
-
-[apache-php]
-recipe = slapos.cookbook:apachephp
-source = ${application:location}
-template = ${application-template:location}/${application-template:filename}
-configuration = ${application-configuration:location}
-
-htdocs = $${rootdirectory:srv}/www/
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-url = http://[$${:ip}]:$${:port}/
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-php-ini-dir = $${directory:php-ini-dir}
-tmp-dir = $${directory:tmp-php}
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
 
-httpd-binary = ${apache:location}/bin/httpd
+# Creation of all needed directories
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
 
-mysql-username = $${mariadb-urlparse:username}
-mysql-password = $${mariadb-urlparse:password}
-mysql-database = $${mariadb-urlparse:path}
-mysql-host = $${stunnel:local-host}
-mysql-port = $${stunnel:local-port}
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
 
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${directory:ca-dir}
-requests-directory = $${cadirectory:requests}
-wrapper = $${basedirectory:services}/ca
-ca-private = $${cadirectory:private}
-ca-certs = $${cadirectory:certs}
-ca-newcerts = $${cadirectory:newcerts}
-ca-crl = $${cadirectory:crl}
+[directory]
+recipe = slapos.cookbook:mkdirectory
+ca-dir = $${rootdirectory:srv}/ssl/
+httpd-log = $${basedirectory:log}/apache/
+php-ini-dir = $${rootdirectory:etc}/php/
+tmp-php = $${rootdirectory:tmp}/php/
+logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+logrotate-backup = $${basedirectory:backup}/logrotate/
+stunnel-conf = $${rootdirectory:etc}/stunnel/
+cronstamps = $${rootdirectory:etc}/cronstamps/
+cron-entries = $${rootdirectory:etc}/cron.d/
+crontabs = $${rootdirectory:etc}/crontabs/
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -83,14 +55,8 @@ certs = $${directory:ca-dir}/certs/
 newcerts = $${directory:ca-dir}/newcerts/
 crl = $${directory:ca-dir}/crl/
 
-[ca-stunnel]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-executable = $${stunnel:wrapper}
-wrapper = $${basedirectory:services}/stunnel
-key-file = $${stunnel:key-file}
-cert-file = $${stunnel:cert-file}
 
+# Deploy stunnel
 [stunnel]
 recipe = slapos.cookbook:stunnel
 client = true
@@ -107,6 +73,29 @@ pid-file = $${basedirectory:run}/stunnel.pid
 wrapper = $${rootdirectory:bin}/raw_stunnel
 post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
+
+# Certificate stuffs
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+# Request MariaDB instance and parse its URL
 [request-mariadb]
 <= slap-connection
 recipe = slapos.cookbook:request
@@ -115,17 +104,41 @@ software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
 
-[request-frontend]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Frontend
-# XXX We have hardcoded SR URL here.
-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-slave = true
-config = url
-config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
-return = site_url
+[mariadb-urlparse]
+recipe = slapos.cookbook:urlparse
+url = $${request-mariadb:connection-url}
+
+
+# Deploy Apache + PHP application
+[apache-php]
+recipe = slapos.cookbook:apachephp
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+htdocs = $${rootdirectory:srv}/www/
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+url = http://[$${:ip}]:$${:port}/
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+mysql-username = $${mariadb-urlparse:username}
+mysql-password = $${mariadb-urlparse:password}
+mysql-database = $${mariadb-urlparse:path}
+mysql-host = $${stunnel:local-host}
+mysql-port = $${stunnel:local-port}
 
+
+# Deploy logrotate, cron, configure it
 [logrotate]
 recipe = slapos.cookbook:logrotate
 # Binaries
@@ -161,6 +174,11 @@ notifempty = true
 create = true
 post = $${stunnel:post-rotate-script}
 
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
 [cron]
 recipe = slapos.cookbook:cron
 dcrond-binary = ${dcron:location}/sbin/crond
@@ -170,11 +188,6 @@ cronstamps = $${directory:cronstamps}
 catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
 [cron-entry-logrotate]
 <= cron
 recipe = slapos.cookbook:cron.d
@@ -182,31 +195,42 @@ name = logrotate
 frequency = 0 0 * * *
 command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+# Request frontend
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url custom_domain
+config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
+return = site_url
+config-custom_domain = $${slap-parameter:domain}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-ca-dir = $${rootdirectory:srv}/ssl/
-httpd-log = $${basedirectory:log}/apache/
-php-ini-dir = $${rootdirectory:etc}/php/
-tmp-php = $${rootdirectory:tmp}/php/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
+
+# Publish all instance parameters (url of instance)
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = $${apache-php:url}
+url = $${request-frontend:connection-site_url}
+
+
+# Deploy promises scripts
+[promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/apache
+hostname = $${apache-php:ip}
+port = $${apache-php:port}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+
+
+[slap-parameter]
+# Default value if no domain is specified
+domain =