Imported from https://svn.erp5.org/repos/sanef/trunk/utils/slapos.recipe.bef_erp5/

setup.py

@@ -41,6 +41,7 @@ setup(name=name,
       zip_safe=True,
       entry_points={
         'zc.buildout': [
+          'bef_erp5 = slapos.recipe.bef_erp5:Recipe',
           'build = slapos.recipe.build:Script',
           'buildcmmi = slapos.recipe.build:Cmmi',
           'download = slapos.recipe.download:Recipe',

slapos/recipe/bef_erp5/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import slapos.recipe.erp5
+import os
+import pkg_resources
+import zc.buildout
+import sys
+import netaddr
+
+BEF_TIMEZONE = 'BST'
+
+def validLoopBackAddress(ip):
+  if netaddr.IPAddress(ip).is_loopback():
+    return True
+  else:
+    return False
+
+def validPublicAddress(ip):
+  return not validLoopBackAddress(ip)
+
+class Recipe(slapos.recipe.erp5.Recipe):
+  site_id = 'erp5'
+
+  def getLocalIPv4Address(self):
+    """Returns local IPv4 address available on partition"""
+    # XXX: Lack checking for locality of address
+    if self.development:
+      # XXX: Development superhack.
+      return slapos.recipe.erp5.Recipe.getLocalIPv4Address(self)
+    return self._getIpAddress(validLoopBackAddress)
+
+  def getGlobalIPv6Address(self):
+    """Returns global IPv6 address available on partition"""
+    if self.development:
+      # XXX: Development superhack.
+      return slapos.recipe.erp5.Recipe.getGlobalIPv6Address(self)
+    # XXX: Lack checking for globality of address
+
+    return self._getIpAddress(validPublicAddress)
+
+  def _getZeoClusterDict(self):
+    site_path = '/erp5/'
+    return {
+        '/': (self._requestZeoFileStorage('Zeo Server 1', 'main'),
+          site_path + 'account_module'),
+        site_path + 'portal_activities': (self._requestZeoFileStorage('Zeo Server 1',
+          'portal_activities'), None),
+        site_path + 'task_report_module': (self._requestZeoFileStorage('Zeo Server 1',
+          'task_report_module'), None),
+        site_path + 'video_request_module': (self._requestZeoFileStorage('Zeo Server 1',
+          'video_request_module'), None),
+        site_path + 'bug_module': (self._requestZeoFileStorage('Zeo Server 1',
+          'bug_module'), None),
+        site_path + 'portal_simulation': (self._requestZeoFileStorage('Zeo Server 1',
+          'portal_simulation'), None),
+        site_path + 'notice_transfer_request_module': (self._requestZeoFileStorage(
+          'Zeo Server 1', 'notice_transfer_request_module'), None),
+        site_path + 'accounting_module': (self._requestZeoFileStorage('Zeo Server 1',
+          'accounting_module'), None),
+        site_path + 'task_module': (self._requestZeoFileStorage('Zeo Server 1',
+          'task_module'), None),
+
+        site_path + 'video_module': (self._requestZeoFileStorage('Zeo Server 2',
+          'video_module'), None),
+
+        site_path + 'event_module': (self._requestZeoFileStorage('Zeo Server 3',
+          'event_module'), None),
+        site_path + 'support_request_module': (self._requestZeoFileStorage(
+          'Zeo Server 3', 'support_request_module'), None),
+
+        site_path + 'person_module': (self._requestZeoFileStorage('Zeo Server 4',
+          'person_module'), None),
+        site_path + 'organisation_module': (self._requestZeoFileStorage('Zeo Server 4',
+          'organisation_module'), None),
+        site_path + 'portal_synchronizations': (self._requestZeoFileStorage(
+          'Zeo Server 4', 'portal_synchronizations'), None),
+
+        site_path + 'scanned_document_module': (self._requestZeoFileStorage(
+          'Zeo Server 5', 'scanned_document_module'), None),
+        site_path + 'item_module': (self._requestZeoFileStorage('Zeo Server 5',
+          'item_module'), None),
+        site_path + 'document_module': (self._requestZeoFileStorage('Zeo Server 5',
+          'document_module'), None),
+        site_path + 'image_module': (self._requestZeoFileStorage('Zeo Server 5',
+          'image_module'), None),
+        site_path + 'external_source_module': (self._requestZeoFileStorage(
+          'Zeo Server 5', 'external_source_module'), None),
+    }
+
+  def installProductionMysql(self):
+    mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678,
+        template_filename=pkg_resources.resource_filename(__name__,
+          'template/my.cnf.in'), parallel_test_database_amount=0,
+          mysql_conf=dict(innodb_buffer_pool_size='10G'))
+    self.setConnectionDict(dict(
+      mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf,
+    ))
+    return self.path_list
+
+  def installProductionApplication(self):
+    site_check_path = '/%s/getId' % self.site_id
+    ca_conf = self.installCertificateAuthority()
+    backend_key, backend_certificate = self.requestCertificate('Login Based Access')
+    memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
+        port=11000)
+    conversion_server_conf = self.installConversionServer(
+        self.getLocalIPv4Address(), 23000, 23060)
+    user, password = self.installERP5()
+    ip = self.getLocalIPv4Address()
+    mount_point_zeo_dict = self._getZeoClusterDict()
+    zeo_conf = self.installZeo(ip)
+    zodb_configuration_list = []
+    known_tid_storage_identifier_dict = {}
+    for mount_point, (storage_dict, check_path) in mount_point_zeo_dict.iteritems():
+      known_tid_storage_identifier_dict[
+        (((storage_dict['ip'],storage_dict['port']),), storage_dict['storage_name'])
+        ] = (zeo_conf[storage_dict['storage_name']]['path'], check_path or mount_point)
+      zodb_configuration_list.append(self.substituteTemplate(
+        self.getTemplateFilename('zope-zeo-snippet.conf.in'), dict(
+        storage_name=storage_dict['storage_name'],
+        address='%s:%s' % (storage_dict['ip'], storage_dict['port']),
+        mount_point=mount_point
+        )))
+    tidstorage_config = dict(host=self.getLocalIPv4Address(), port='6001')
+    zodb_configuration_string = '\n'.join(zodb_configuration_list)
+    zope_port = 12000
+    # One Distribution Node
+    zope_port +=1
+    self.installZope(ip, zope_port, 'zope_distribution',
+        with_timerservice=True,
+        zodb_configuration_string=zodb_configuration_string,
+        tidstorage_config=tidstorage_config,
+        zope_environment=dict(TZ=BEF_TIMEZONE))
+    # Two Activity Nodes
+    for i in (1, 2):
+      zope_port += 1
+      self.installZope(ip, zope_port, 'zope_activity_%s' % i,
+        with_timerservice=True,
+        zodb_configuration_string=zodb_configuration_string,
+        tidstorage_config=tidstorage_config,
+        zope_environment=dict(TZ=BEF_TIMEZONE))
+    # Eight Working Nodes (Human access)
+    login_url_list = []
+    for i in (1, 2, 3, 4, 5, 6, 7, 8):
+      zope_port += 1
+      login_url_list.append(self.installZope(ip, zope_port,
+        'zope_login_%s' % i, with_timerservice=False,
+        zodb_configuration_string=zodb_configuration_string,
+        tidstorage_config=tidstorage_config,
+        zope_environment=dict(TZ=BEF_TIMEZONE)))
+    login_haproxy = self.installHaproxy(ip, 15001, 'login', site_check_path,
+        login_url_list)
+    # One Web Node
+    zope_port += 1
+    web_url_list = [self.installZope(ip, zope_port, 'zope_web',
+        with_timerservice=True,
+        zodb_configuration_string=zodb_configuration_string,
+        tidstorage_config=tidstorage_config,
+        thread_amount=10, zope_environment=dict(TZ=BEF_TIMEZONE))]
+    web_haproxy = self.installHaproxy(ip, 15002, 'web', site_check_path,
+        web_url_list)
+    apache_web = self.installBackendApache(self.getGlobalIPv6Address(), 15001,
+        web_haproxy, backend_key, backend_certificate, suffix='_web')
+
+    # One Admin Node
+    zope_port += 1
+    admin_url_list = [self.installZope(ip, zope_port, 'zope_admin',
+        with_timerservice=True,
+        zodb_configuration_string=zodb_configuration_string,
+        tidstorage_config=tidstorage_config,
+        zope_environment=dict(TZ=BEF_TIMEZONE))]
+    admin_haproxy = self.installHaproxy(ip, 15003, 'admin', site_check_path,
+        admin_url_list)
+    apache_admin = self.installBackendApache(self.getGlobalIPv6Address(), 15002,
+        admin_haproxy, backend_key, backend_certificate, suffix='_admin')
+
+    self.installTidStorage(tidstorage_config['host'], tidstorage_config['port'],
+        known_tid_storage_identifier_dict, 'http://'+login_haproxy)
+    apache_login = self.installBackendApache(self.getGlobalIPv6Address(), 15000,
+        login_haproxy, backend_key, backend_certificate,
+        access_control_string='%s' % self.getGlobalIPv6Address())
+    frontend_key, frontend_certificate = self.requestCertificate(
+        self.parameter_dict['frontend_name'])
+    login_frontend = self.installFrontendZopeApache(
+        self.getGlobalIPv6Address(), 18000,
+        self.parameter_dict['frontend_name'],
+        self.parameter_dict['frontend_path'],
+        apache_login,
+        self.parameter_dict['backend_path'], frontend_key,
+        frontend_certificate,
+        access_control_string=self.parameter_dict['frontend_acl_string'])
+    memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
+        port=11000)
+    kumo_conf = self.installKumo(self.getLocalIPv4Address())
+    self.setConnectionDict(dict(
+      site_url='https://%s:%s%s' % (self.getGlobalIPv6Address(), 18000,
+        self.parameter_dict['frontend_path']),
+      site_web_url=apache_web,
+      site_admin_url=apache_admin,
+      site_user=user,
+      site_password=password,
+      memcached_url=memcached_conf['memcached_url'],
+      kumo_url=kumo_conf['kumo_address'],
+      conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' %
+        conversion_server_conf,
+      # openssl binary might be removed, as soon as CP environment will be
+      # fully controlled
+      openssl_binary=self.options['openssl_binary'],
+      # As soon as there would be Vifib ERP5 configuration and possibility to
+      # call it over the network this can be removed
+      certificate_authority_path=ca_conf['certificate_authority_path'],
+    ))
+    return self.path_list
+
+  def installDevelopmentEnvironment(self):
+    ca_conf = self.installCertificateAuthority()
+    memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
+        port=11000)
+    conversion_server_conf = self.installConversionServer(
+        self.getLocalIPv4Address(), 23000, 23060)
+    mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678,
+        template_filename=pkg_resources.resource_filename(__name__,
+          'template/my.cnf.in'), parallel_test_database_amount=0,
+          mysql_conf=dict(innodb_buffer_pool_size='1G'))
+    kumo_conf = self.installKumo(self.getLocalIPv4Address())
+    user, password = self.installERP5()
+    self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf,
+        memcached_conf, kumo_conf)
+    ip = self.getLocalIPv4Address()
+    zope_port = '18080'
+    zodb_dir = os.path.join(self.data_root_directory, 'zodb')
+    self._createDirectory(zodb_dir)
+    zodb_root_path = os.path.join(zodb_dir, 'root.fs')
+    self.installZope(ip, zope_port, 'zope_development',
+        zodb_configuration_string=self.substituteTemplate(
+          self.getTemplateFilename('zope-zodb-snippet.conf.in'),
+          dict(zodb_root_path=zodb_root_path)),
+          thread_amount=8, with_timerservice=True,
+          zope_environment=dict(TZ=BEF_TIMEZONE))
+    self.setConnectionDict(dict(
+      site_user=user,
+      site_password=password,
+      memcached_url=memcached_conf['memcached_url'],
+      kumo_url=kumo_conf['kumo_address'],
+      conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' %
+        conversion_server_conf,
+      # openssl binary might be removed, as soon as CP environment will be
+      # fully controlled
+      openssl_binary=self.options['openssl_binary'],
+      # As soon as there would be Vifib ERP5 configuration and possibility to
+      # call it over the network this can be removed
+      certificate_authority_path=ca_conf['certificate_authority_path'],
+      # as installERP5Site is not trusted (yet) and this recipe is production
+      # ready expose more information
+      mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf,
+      development_zope='http://%s:%s/' % (ip, zope_port)
+    ))
+    return self.path_list
+
+  def _install(self):
+    self.path_list = []
+    self.requirements, self.ws = self.egg.working_set([__name__])
+    # self.cron_d is a directory, where cron jobs can be registered
+    self.cron_d = self.installCrond()
+    self.logrotate_d, self.logrotate_backup = self.installLogrotate()
+    self.killpidfromfile = zc.buildout.easy_install.scripts(
+        [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
+          'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
+    self.path_list.append(self.killpidfromfile)
+    self.linkBinary()
+    if self.parameter_dict.get('development', 'false').lower() == 'true':
+      self.development = True
+      return self.installDevelopmentEnvironment()
+    if self.parameter_dict.get('production_mysql', 'false').lower() == 'true':
+      self.development = False
+      return self.installProductionMysql()
+    if self.parameter_dict.get('production_application', 'false').lower() == 'true':
+      self.development = False
+      return self.installProductionApplication()
+
+    raise NotImplementedError('Flavour of instance have to be given.')

slapos/recipe/bef_erp5/template/my.cnf.in

+# Example MySQL config file for medium systems.
+#
+# This is for a system with little memory (32M - 64M) where MySQL plays
+# an important part, or systems up to 128M where MySQL is used together with
+# other programs (such as a web server)
+#
+
+# The following options will be passed to all MySQL clients
+[client]
+user		= root
+#password	= your_password
+port		= %(tcp_port)s
+socket		= %(socket)s
+
+# Here follows entries for some specific programs
+
+# The MySQL server
+[mysqld]
+#user		= mysql # not needed in SlapOS
+datadir		= %(data_directory)s
+port		= %(tcp_port)s
+socket		= %(socket)s
+pid-file	= %(pid_file)s
+skip-locking
+key_buffer = 128M
+max_allowed_packet = 100M
+table_cache = 1024
+sort_buffer_size = 100M
+net_buffer_length = 8K
+read_buffer_size = 256K
+read_rnd_buffer_size = 512K
+myisam_sort_buffer_size = 8M
+collation_server = utf8_unicode_ci
+character_set_server = utf8
+
+log-slow-queries=%(slow_query_log)s
+long_query_time=1
+
+bind-address = %(ip)s # new in SlapOS
+log-error = %(error_log)s # new in SlapOS
+
+# max_connections = 256
+max_connections = 512
+# Default to using old password format for compatibility with old and
+# shorter password hash.
+# Reference: http://dev.mysql.com/doc/mysql/en/Password_hashing.html
+old_passwords
+
+# Don't listen on a TCP/IP port at all. This can be a security enhancement,
+# if all processes that need to connect to mysqld run on the same host.
+# All interaction with mysqld must be made via Unix sockets or named pipes.
+# Note that using this option without enabling named pipes on Windows
+# (via the "enable-named-pipe" option) will render mysqld useless!
+# 
+# skip-networking
+
+# Replication Master Server (default)
+# binary logging is required for replication
+#log-bin=mysql-bin
+
+# required unique id between 1 and 2^32 - 1
+# defaults to 1 if master-host is not set
+# but will not function as a master if omitted
+server-id	= 1
+
+# Replication Slave (comment out master section to use this)
+#
+# To configure this host as a replication slave, you can choose between
+# two methods :
+#
+# 1) Use the CHANGE MASTER TO command (fully described in our manual) -
+#    the syntax is:
+#
+#    CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>,
+#    MASTER_USER=<user>, MASTER_PASSWORD=<password> ;
+#
+#    where you replace <host>, <user>, <password> by quoted strings and
+#    <port> by the master's port number (%(tcp_port)s by default).
+#
+#    Example:
+#
+#    CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=%(tcp_port)s,
+#    MASTER_USER='joe', MASTER_PASSWORD='secret';
+#
+# OR
+#
+# 2) Set the variables below. However, in case you choose this method, then
+#    start replication for the first time (even unsuccessfully, for example
+#    if you mistyped the password in master-password and the slave fails to
+#    connect), the slave will create a master.info file, and any later
+#    change in this file to the variables' values below will be ignored and
+#    overridden by the content of the master.info file, unless you shutdown
+#    the slave server, delete master.info and restart the slaver server.
+#    For that reason, you may want to leave the lines below untouched
+#    (commented) and instead use CHANGE MASTER TO (see above)
+#
+# required unique id between 2 and 2^32 - 1
+# (and different from the master)
+# defaults to 2 if master-host is set
+# but will not function as a slave if omitted
+#server-id       = 2
+#
+# The replication master for this slave - required
+#master-host     =   <hostname>
+#
+# The username the slave will use for authentication when connecting
+# to the master - required
+#master-user     =   <username>
+#
+# The password the slave will authenticate with when connecting to
+# the master - required
+#master-password =   <password>
+#
+# The port the master is listening on.
+# optional - defaults to %(tcp_port)s
+#master-port     =  <port>
+#
+# binary logging - not required for slaves, but recommended
+#log-bin=mysql-bin
+
+# Point the following paths to different dedicated disks
+#tmpdir		= /tmp/		
+#log-update 	= /path-to-dedicated-directory/hostname
+
+# Uncomment the following if you are using BDB tables
+#bdb_cache_size = 4M
+#bdb_max_lock = 10000
+
+# Uncomment the following if you are using InnoDB tables
+#innodb_data_home_dir = /var/lib/mysql/
+#innodb_data_file_path = ibdata1:10M:autoextend
+#innodb_log_group_home_dir = /var/lib/mysql/
+#innodb_log_arch_dir = /var/lib/mysql/
+# You can set .._buffer_pool_size up to 50 - 80 %%
+# of RAM but beware of setting memory usage too high
+innodb_buffer_pool_size = %(innodb_buffer_pool_size)s
+innodb_additional_mem_pool_size = 40M
+# Set .._log_file_size to 25 %% of buffer pool size
+#innodb_log_file_size = 5M
+#innodb_log_buffer_size = 8M
+#innodb_flush_log_at_trx_commit = 1
+#innodb_lock_wait_timeout = 50
+
+innodb_locks_unsafe_for_binlog=1
+
+#bind-address=192.168.100.1
+
+## Options for mysqld process:
+#ndbcluster                      # run NDB engine
+#ndb-connectstring=192.168.0.10  # location of MGM node
+
+## Options for ndbd process:
+#[mysql_cluster]                 
+#ndb-connectstring=192.168.0.10  # location of MGM node
+#
+
+# Use utf8 as the default character set.                                                                                                                                                                       
+default-character-set=utf8                                                                                                                                                                                     
+# Ignore client information and use the default server character set.                                                                                                                                          
+skip-character-set-client-handshake
+
+
+[mysqldump]
+quick
+max_allowed_packet = 16M
+
+[mysql]
+no-auto-rehash
+# Remove the next comment character if you are not familiar with SQL
+#safe-updates
+default-character-set = utf8
+
+[isamchk]
+key_buffer = 20M
+sort_buffer_size = 20M
+read_buffer = 2M
+write_buffer = 2M
+
+[myisamchk]
+key_buffer = 20M
+sort_buffer_size = 20M
+read_buffer = 2M
+write_buffer = 2M
+
+[mysqlhotcopy]
+interactive-timeout