Add osoeslaptraining recipe to cookbook

slapos/recipe/osoeslaptraining/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import BaseSlapRecipe
+import os
+import shutil
+import pkg_resources
+import zc.buildout
+import sys
+import zc.recipe.egg
+
+class BaseRecipe(BaseSlapRecipe):
+  def __init__(self, buildout, name, options):
+    BaseSlapRecipe.__init__(self, buildout, name, options)
+    options['eggs'] = 'slapos.recipe.osoeslaptraining'
+    self.egg = zc.recipe.egg.Scripts(buildout, name, options)
+
+  def installMysqlServer(self, ip=None, port=None):
+    if ip is None:
+      ip = self.getLocalIPv4Address()
+    if port is None:
+      port = '3306'
+    mysql_conf = dict(
+        ip=ip,
+        data_directory=os.path.join(self.data_root_directory,
+          'mysql'),
+        tcp_port=port,
+        pid_file=os.path.join(self.run_directory, 'mysqld.pid'),
+        socket=os.path.join(self.run_directory, 'mysqld.sock'),
+        error_log=os.path.join(self.log_directory, 'mysqld.log'),
+        slow_query_log=os.path.join(self.log_directory,
+        'mysql-slow.log'),
+        database='appdb',
+        user='appuser',
+        password=self.generatePassword(),
+    )
+    self._createDirectory(mysql_conf['data_directory'])
+
+    mysql_conf_path = self.createConfigurationFile("my.cnf",
+        self.substituteTemplate(pkg_resources.resource_filename(__name__, 'template/my.cnf.in'),
+          mysql_conf))
+
+    mysql_script = pkg_resources.resource_string(__name__,
+        'template/mysqlinit.sql.in') % mysql_conf
+    self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update',
+      __name__ + '.mysql', 'updateMysql')], self.ws,
+      sys.executable, self.wrapper_directory, arguments=[dict(
+        mysql_script=mysql_script,
+        mysql_binary=self.options['mysql_binary'].strip(),
+        mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(),
+        socket=mysql_conf['socket'],
+        )]))
+    self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld',
+      __name__ + '.mysql', 'runMysql')], self.ws,
+        sys.executable, self.wrapper_directory, arguments=[dict(
+        mysql_install_binary=self.options['mysql_install_binary'].strip(),
+        mysqld_binary=self.options['mysqld_binary'].strip(),
+        data_directory=mysql_conf['data_directory'].strip(),
+        mysql_binary=self.options['mysql_binary'].strip(),
+        socket=mysql_conf['socket'].strip(),
+        configuration_file=mysql_conf_path,
+       )]))
+    self.path_list.extend([mysql_conf_path])
+    return dict(
+      mysql_host=mysql_conf['ip'],
+      mysql_port=mysql_conf['tcp_port'],
+      mysql_user=mysql_conf['user'],
+      mysql_password=mysql_conf['password'],
+      mysql_database=mysql_conf['database'],
+    )
+
+  def createHtdocs(self, source, document_root):
+    source = self.options['source'].strip()
+    document_root = self.createDataDirectory('htdocs')
+    for p in os.listdir(document_root):
+      path = os.path.join(document_root, p)
+      if os.path.isdir(path):
+        shutil.rmtree(path)
+      else:
+        os.unlink(path)
+    for p in os.listdir(source):
+      path = os.path.join(source, p)
+      if os.path.isdir(path):
+        shutil.copytree(path, os.path.join(document_root, p))
+      else:
+        shutil.copy2(path, os.path.join(document_root, p))
+
+  def installApache(self, document_root, ip=None, port=None):
+    if ip is None:
+      ip=self.getGlobalIPv6Address()
+    if port is None:
+      port = '9080'
+    apache_config = dict(
+        pid_file=os.path.join(self.run_directory, 'httpd.pid'),
+        lock_file=os.path.join(self.run_directory, 'httpd.lock'),
+        ip=ip,
+        port=port,
+        error_log=os.path.join(self.log_directory, 'httpd-error.log'),
+        access_log=os.path.join(self.log_directory, 'httpd-access.log'),
+        document_root=document_root,
+        php_ini_dir=self.etc_directory
+    )
+    config_file = self.createConfigurationFile('httpd.conf',
+        self.substituteTemplate(pkg_resources.resource_filename(__name__,
+          'template/apache.in'), apache_config))
+    self.path_list.append(config_file)
+    self.path_list.append(self.createConfigurationFile('php.ini',
+        self.substituteTemplate(pkg_resources.resource_filename(__name__,
+          'template/php.ini.in'), {})))
+    self.path_list.extend(zc.buildout.easy_install.scripts([(
+      'httpd',
+        __name__ + '.apache', 'runApache')], self.ws,
+          sys.executable, self.wrapper_directory, arguments=[
+            dict(
+              required_path_list=[],
+              binary=self.options['httpd_binary'],
+              config=config_file
+            )
+          ]))
+    return 'http://[%s]:%s' % (ip, port)
+
+  def createConfiguration(self, template, document_root, destination, d):
+    directory = os.path.dirname(destination)
+    file = os.path.basename(destination)
+    path = document_root
+    if directory:
+      path = os.path.join(document_root, directory)
+      if not os.path.exists(path):
+        os.makedirs(path)
+    destination = os.path.join(path, file)
+    open(destination, 'w').write(open(template, 'r').read() % d)
+
+class Static(BaseRecipe):
+  def _install(self):
+    self.path_list = []
+    self.requirements, self.ws = self.egg.working_set()
+    document_root = self.createDataDirectory('htdocs')
+    self.createHtdocs(self.options['source'].strip(), document_root)
+    url = self.installApache(document_root)
+    self.setConnectionDict(dict(url = url))
+    return self.path_list
+
+class Simple(BaseRecipe):
+  def _install(self):
+    self.path_list = []
+    self.requirements, self.ws = self.egg.working_set()
+    document_root = self.createDataDirectory('htdocs')
+    self.createHtdocs(self.options['source'].strip(), document_root)
+    mysql_conf = self.installMysqlServer()
+    url = self.installApache(document_root)
+    self.setConnectionDict(dict(
+      url=url,
+      **mysql_conf
+    ))
+    self.createConfiguration(self.options['template'], document_root,
+        self.options['configuration'], mysql_conf)
+    return self.path_list
+
+class Request(BaseRecipe):
+  def _install(self):
+    self.path_list = []
+    self.requirements, self.ws = self.egg.working_set()
+    software_type = self.parameter_dict['slap_software_type']
+    if software_type == 'RootSoftwareInstance':
+      document_root = self.createDataDirectory('htdocs')
+      self.createHtdocs(self.options['source'].strip(), document_root)
+      mysql = self.request(self.software_release_url, 'MySQL Server', 'mysql')
+      mysql_conf = dict(
+        mysql_host=mysql.getConnectionParameter('mysql_host'),
+        mysql_port=mysql.getConnectionParameter('mysql_port'),
+        mysql_user=mysql.getConnectionParameter('mysql_user'),
+        mysql_password=mysql.getConnectionParameter('mysql_password'),
+        mysql_database=mysql.getConnectionParameter('mysql_database'),
+      )
+      url = self.installApache(document_root)
+      self.setConnectionDict(dict(
+        url=url,
+      ))
+      self.createConfiguration(self.options['template'], document_root,
+          self.options['configuration'], mysql_conf)
+    elif software_type == 'MySQL Server':
+      mysql_conf = self.installMysqlServer()
+      self.setConnectionDict(dict(
+        **mysql_conf
+      ))
+    else:
+      raise zc.buildout.UserError('Uknown software type %r' % software_type)
+    return self.path_list

slapos/recipe/osoeslaptraining/apache.py

+import os
+import sys
+import time
+
+
+def runApache(args):
+  sleep = 60
+  conf = args[0]
+  while True:
+    ready = True
+    for f in conf.get('required_path_list', []):
+      if not os.path.exists(f):
+        print 'File %r does not exists, sleeping for %s' % (f, sleep)
+        ready = False
+    if ready:
+      break
+    time.sleep(sleep)
+  apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND']
+  apache_wrapper_list.extend(sys.argv[1:])
+  sys.stdout.flush()
+  sys.stderr.flush()
+  os.execl(apache_wrapper_list[0], *apache_wrapper_list)

slapos/recipe/osoeslaptraining/mysql.py

+import os
+import subprocess
+import time
+import sys
+
+
+def runMysql(args):
+  sleep = 60
+  conf = args[0]
+  mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
+      conf['configuration_file']]
+  # we trust mysql_install that if mysql directory is available mysql was
+  # correctly initalised
+  if not os.path.isdir(os.path.join(conf['data_directory'], 'mysql')):
+    while True:
+      # XXX: Protect with proper root password
+      popen = subprocess.Popen([conf['mysql_install_binary'],
+        '--skip-name-resolve', '--no-defaults', '--datadir=%s' %
+        conf['data_directory']],
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      result = popen.communicate()[0]
+      if popen.returncode is None or popen.returncode != 0:
+        print "Failed to initialise server.\nThe error was: %s" % result
+        print "Waiting for %ss and retrying" % sleep
+        time.sleep(sleep)
+      else:
+        print "Mysql properly initialised"
+        break
+  else:
+    print "MySQL already initialised"
+  print "Starting %r" % mysqld_wrapper_list[0]
+  sys.stdout.flush()
+  sys.stderr.flush()
+  os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
+
+
+def updateMysql(args):
+  conf = args[0]
+  sleep = 30
+  is_succeed = False
+  while True:
+    if not is_succeed:
+      mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
+      mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      result = mysql_upgrade.communicate()[0]
+      if mysql_upgrade.returncode is None:
+        mysql_upgrade.kill()
+      if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
+        print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
+        print 'Sleeping for %ss and retrying' % sleep
+      else:
+        if mysql_upgrade.returncode == 0:
+          print "MySQL database upgraded with result:\n%s" % result
+        else:
+          print "No need to upgrade MySQL database"
+        mysql_script = conf.get('mysql_script')
+        if mysql_script:
+          mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
+          mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
+              stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          result = mysql.communicate(conf['mysql_script'])[0]
+          if mysql.returncode is None:
+            mysql.kill()
+          if mysql.returncode != 0:
+            print 'Command %r failed with:\n%s' % (mysql_list, result)
+            print 'Sleeping for %ss and retrying' % sleep
+          else:
+            is_succeed = True
+            print 'SlapOS initialisation script succesfully applied on database.'
+    sys.stdout.flush()
+    sys.stderr.flush()
+    time.sleep(sleep)

slapos/recipe/osoeslaptraining/template/apache.in

+# Apache static configuration
+# Automatically generated
+
+# Basic server configuration
+PidFile "%(pid_file)s"
+LockFile "%(lock_file)s"
+Listen %(ip)s:%(port)s
+PHPINIDir %(php_ini_dir)s
+ServerAdmin someone@email
+DefaultType text/plain
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+AddType application/x-httpd-php .php .phtml .php5 .php4
+AddType application/x-httpd-php-source .phps
+
+# Log configuration
+ErrorLog "%(error_log)s"
+LogLevel warn
+LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined
+LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
+CustomLog "%(access_log)s" common
+
+# Directory protection
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+    Order deny,allow
+    Deny from all
+</Directory>
+
+<Directory %(document_root)s>
+  Options FollowSymLinks
+  AllowOverride All
+  Order allow,deny
+  Allow from all
+</Directory>
+DocumentRoot %(document_root)s
+DirectoryIndex index.html index.php
+
+# List of modules
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule dir_module modules/mod_dir.so
+LoadModule php5_module modules/libphp5.so

slapos/recipe/osoeslaptraining/template/my.cnf.in

+# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
+# The MySQL server
+[mysqld]
+# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
+# different engine, like MyISAM. Such behaviour generates problems only, when
+# tables requested as InnoDB are silently created with MyISAM engine.
+#
+# Loud fail is really required in such case.
+sql-mode="NO_ENGINE_SUBSTITUTION"
+
+skip-show-database
+port = %(tcp_port)s
+bind-address = %(ip)s
+socket = %(socket)s
+datadir = %(data_directory)s
+pid-file = %(pid_file)s
+log-error = %(error_log)s
+log-slow-file = %(slow_query_log)s
+long_query_time = 5
+max_allowed_packet = 128M
+query_cache_size = 32M
+
+plugin-load = ha_innodb_plugin.so
+
+# The following are important to configure and depend a lot on to the size of
+# your database and the available resources.
+#innodb_buffer_pool_size = 4G
+#innodb_log_file_size = 256M
+#innodb_log_buffer_size = 8M
+
+# Some dangerous settings you may want to uncomment if you only want
+# performance or less disk access. Useful for unit tests.
+#innodb_flush_log_at_trx_commit = 0
+#innodb_flush_method = nosync
+#innodb_doublewrite = 0
+#sync_frm = 0
+
+# Uncomment the following if you need binary logging, which is recommended
+# on production instances (either for replication or incremental backups).
+#log-bin=mysql-bin
+
+# Force utf8 usage
+collation_server = utf8_unicode_ci
+character_set_server = utf8
+skip-character-set-client-handshake
+
+[mysql]
+no-auto-rehash
+socket = %(socket)s
+
+[mysqlhotcopy]
+interactive-timeout

slapos/recipe/osoeslaptraining/template/mysqlinit.sql.in

+CREATE DATABASE IF NOT EXISTS %(database)s;
+GRANT ALL PRIVILEGES ON %(database)s.* TO %(user)s@localhost IDENTIFIED BY %(password)r;
+GRANT ALL PRIVILEGES ON %(database)s.* TO %(user)s@'%%' IDENTIFIED BY %(password)r;
+GRANT SHOW DATABASES ON *.* TO %(user)s@localhost IDENTIFIED BY %(password)r;
+GRANT SHOW DATABASES ON *.* TO %(user)s@'%%' IDENTIFIED BY %(password)r;
+FLUSH PRIVILEGES;
+EXIT

slapos/recipe/osoeslaptraining/template/php.ini.in

+[PHP]
+engine = On
+safe_mode = Off
+expose_php = On
+error_reporting = E_ALL | E_STRICT
+display_errors = On
+display_startup_errors = On
+log_errors = On
+log_errors_max_len = 1024
+ignore_repeated_errors = Off
+ignore_repeated_source = Off