Commit a7eaea51 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge branch 'master' into erp5-component

parents 492ee3a1 e6cce9da
...@@ -15,7 +15,8 @@ extends = ...@@ -15,7 +15,8 @@ extends =
../mariadb/buildout.cfg ../mariadb/buildout.cfg
../openldap/buildout.cfg ../openldap/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../zip/buildout.cfg ../zlib/buildout.cfg
[apache-php] [apache-php]
# Note: Shall react on each build of apache and reinstall itself # Note: Shall react on each build of apache and reinstall itself
...@@ -29,18 +30,16 @@ configure-options = ...@@ -29,18 +30,16 @@ configure-options =
--with-libxml-dir=${libxml2:location} --with-libxml-dir=${libxml2:location}
--with-mysql=${mariadb:location} --with-mysql=${mariadb:location}
--with-zlib-dir=${zlib:location} --with-zlib-dir=${zlib:location}
--with-bz2-dir=${bzip2:location} --with-bz2=${bzip2:location}
--with-mcrypt=${libmcrypt:location} --with-mcrypt=${libmcrypt:location}
--with-gd --with-gd
--with-jpeg-dir=${libjpeg:location} --with-jpeg-dir=${libjpeg:location}
--with-png-dir=${libpng:location} --with-png-dir=${libpng:location}
--enable-gd-native-ttf --enable-gd-native-ttf
--with-ttf
--with-freetype-dir=${freetype:location} --with-freetype-dir=${freetype:location}
--with-pdo-mysql=mysqlnd --with-pdo-mysql=mysqlnd
--with-mysqli=mysqlnd --with-mysqli=mysqlnd
--with-curl=${curl:location} --with-curl=${curl:location}
--with-zip-dir=${zip:location}
--with-imap=${cclient:location} --with-imap=${cclient:location}
--with-iconv-dir=${libiconv:location} --with-iconv-dir=${libiconv:location}
--with-gettext=${gettext:location} --with-gettext=${gettext:location}
...@@ -52,10 +51,8 @@ configure-options = ...@@ -52,10 +51,8 @@ configure-options =
--enable-session --enable-session
--enable-exif --enable-exif
--enable-zip --enable-zip
--enable-bz2
--enable-ftp --enable-ftp
# Changing TMPDIR is required for PEAR installation. # Changing TMPDIR is required for PEAR installation.
# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp. # It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
# XXX we could mkdir tmp there # XXX we could mkdir tmp there
...@@ -69,23 +66,25 @@ environment = ...@@ -69,23 +66,25 @@ environment =
[apache-php-postgres] [apache-php-postgres]
<=apache-php # Note: Shall react on each build of apache and reinstall itself
recipe = hexagonit.recipe.cmmi
depends =
${postgresql:url}
url = http://fr2.php.net/distributions/php-5.4.12.tar.bz2
md5sum = 5c7b614242ae12e9cacca21c8ab84818
configure-options = configure-options =
--with-apxs2=${apache:location}/bin/apxs --with-apxs2=${apache:location}/bin/apxs
--with-libxml-dir=${libxml2:location} --with-libxml-dir=${libxml2:location}
--with-zlib-dir=${zlib:location} --with-zlib-dir=${zlib:location}
--with-bz2-dir=${bzip2:location} --with-bz2=${bzip2:location}
--with-mcrypt=${libmcrypt:location} --with-mcrypt=${libmcrypt:location}
--with-gd --with-gd
--with-jpeg-dir=${libjpeg:location} --with-jpeg-dir=${libjpeg:location}
--with-png-dir=${libpng:location} --with-png-dir=${libpng:location}
--enable-gd-native-ttf --enable-gd-native-ttf
--with-ttf
--with-freetype-dir=${freetype:location} --with-freetype-dir=${freetype:location}
--with-curl=${curl:location} --with-curl=${curl:location}
--with-zip-dir=${zip:location}
--with-imap=${cclient:location} --with-imap=${cclient:location}
--with-iconv-dir=${libiconv:location} --with-iconv-dir=${libiconv:location}
--with-gettext=${gettext:location} --with-gettext=${gettext:location}
...@@ -97,10 +96,21 @@ configure-options = ...@@ -97,10 +96,21 @@ configure-options =
--enable-session --enable-session
--enable-exif --enable-exif
--enable-zip --enable-zip
--enable-bz2
--enable-ftp --enable-ftp
--with-pgsql=${postgresql:location} --with-pgsql=${postgresql:location}
# Changing TMPDIR is required for PEAR installation.
# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
# XXX we could mkdir tmp there
environment =
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
HOME=${apache:location}
[libmcrypt] [libmcrypt]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
......
...@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py ...@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py
[stunnel-4] [stunnel-4]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.54.tar.gz url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.55.tar.gz
md5sum = c2b1db99e3ed547214568959a8ed18ac md5sum = ebe6670368d30c6a01ca33b9c50cb239
pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook
configure-options = configure-options =
--enable-ipv6 --enable-ipv6
......
...@@ -85,8 +85,20 @@ class Recipe(BaseSlapRecipe): ...@@ -85,8 +85,20 @@ class Recipe(BaseSlapRecipe):
domain_dict = {} domain_dict = {}
for slave_instance in slave_instance_list: for slave_instance in slave_instance_list:
# Sanitize inputs
backend_url = slave_instance.get("url", None) backend_url = slave_instance.get("url", None)
reference = slave_instance.get("slave_reference") reference = slave_instance.get("slave_reference")
if slave_instance.haskey("enable_cache"):
enable_cache = slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE')
else:
enable_cache = False
if slave_instance.haskey("type"):
slave_type = slave_instance.get("type", "").lower()
else:
slave_type = None
# Set scheme (http? https?) # Set scheme (http? https?)
# Future work may allow to choose between http and https (or both?) # Future work may allow to choose between http and https (or both?)
scheme = 'http://' scheme = 'http://'
...@@ -120,12 +132,14 @@ class Recipe(BaseSlapRecipe): ...@@ -120,12 +132,14 @@ class Recipe(BaseSlapRecipe):
slave_dict[reference] = "%s%s/" % (scheme, domain) slave_dict[reference] = "%s%s/" % (scheme, domain)
# Check if we want varnish+stunnel cache. # Check if we want varnish+stunnel cache.
if slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE'): #if enable_cache:
# XXX-Cedric : need to refactor to clean code? (to many variables) # # XXX-Cedric : need to refactor to clean code? (to many variables)
rewrite_rule = self.configureVarnishSlave( # rewrite_rule = self.configureVarnishSlave(
base_varnish_port, backend_url, reference, service_dict, domain) # base_varnish_port, backend_url, reference, service_dict, domain)
base_varnish_port += 2 # base_varnish_port += 2
else: #else:
# rewrite_rule = "%s %s" % (domain, backend_url)
# Temporary forbid activation of cache until it is properly tested
rewrite_rule = "%s %s" % (domain, backend_url) rewrite_rule = "%s %s" % (domain, backend_url)
# Finally, if successful, we add the rewrite rule to our list of rules # Finally, if successful, we add the rewrite rule to our list of rules
...@@ -134,7 +148,7 @@ class Recipe(BaseSlapRecipe): ...@@ -134,7 +148,7 @@ class Recipe(BaseSlapRecipe):
# rule structure. # rule structure.
# So we will have one RewriteMap for normal websites, and one # So we will have one RewriteMap for normal websites, and one
# RewriteMap for Zope Virtual Host Monster websites. # RewriteMap for Zope Virtual Host Monster websites.
if slave_instance.get("type", "").lower() in ['zope']: if slave_type in ['zope']:
rewrite_rule_zope_list.append(rewrite_rule) rewrite_rule_zope_list.append(rewrite_rule)
# For Zope, we have another dict containing the path e.g '/erp5/... # For Zope, we have another dict containing the path e.g '/erp5/...
rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', '')) rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
...@@ -529,7 +543,7 @@ class Recipe(BaseSlapRecipe): ...@@ -529,7 +543,7 @@ class Recipe(BaseSlapRecipe):
self._createDirectory(mod_ssl_cache_location) self._createDirectory(mod_ssl_cache_location)
# Create "custom" apache configuration files if it does not exist. # Create "custom" apache configuration files if it does not exist.
# Note : Those files won't be erased or changed when slapgrid is ran. # Note : Those files won't be erased or changed by slapgrid.
# It can be freely customized by node admin. # It can be freely customized by node admin.
custom_apache_configuration_directory = os.path.join( custom_apache_configuration_directory = os.path.join(
self.data_root_directory, 'apache-conf.d') self.data_root_directory, 'apache-conf.d')
...@@ -537,12 +551,14 @@ class Recipe(BaseSlapRecipe): ...@@ -537,12 +551,14 @@ class Recipe(BaseSlapRecipe):
# First one is included in the end of the apache configuration file # First one is included in the end of the apache configuration file
custom_apache_configuration_file_location = os.path.join( custom_apache_configuration_file_location = os.path.join(
custom_apache_configuration_directory, 'apache_frontend.custom.conf') custom_apache_configuration_directory, 'apache_frontend.custom.conf')
open(custom_apache_configuration_file_location, 'a') if not os.path.exists(custom_apache_configuration_file_location):
open(custom_apache_configuration_file_location, 'w')
# Second one is included in the virtualhost of apache configuration file # Second one is included in the virtualhost of apache configuration file
custom_apache_virtual_configuration_file_location = os.path.join( custom_apache_virtual_configuration_file_location = os.path.join(
custom_apache_configuration_directory, custom_apache_configuration_directory,
'apache_frontend.virtualhost.custom.conf') 'apache_frontend.virtualhost.custom.conf')
open(custom_apache_virtual_configuration_file_location, 'a') if not os.path.exists(custom_apache_virtual_configuration_file_location):
open(custom_apache_virtual_configuration_file_location, 'w')
# Create backup of custom apache configuration # Create backup of custom apache configuration
backup_path = self.createBackupDirectory('custom_apache_conf_backup') backup_path = self.createBackupDirectory('custom_apache_conf_backup')
......
...@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe): ...@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
'# TYPE DATABASE USER ADDRESS METHOD', '# TYPE DATABASE USER ADDRESS METHOD',
'', '',
'# "local" is for Unix domain socket connections only (check unix_socket_permissions!)', '# "local" is for Unix domain socket connections only (check unix_socket_permissions!)',
'local all all ident', 'local all all trust',
'host all all 127.0.0.1/32 md5', 'host all all 127.0.0.1/32 md5',
'host all all ::1/128 md5', 'host all all ::1/128 md5',
] ]
......
...@@ -63,6 +63,7 @@ class ExportRecipe(GenericBaseRecipe): ...@@ -63,6 +63,7 @@ class ExportRecipe(GenericBaseRecipe):
umask 077 umask 077
%(bin)s/pg_dump \\ %(bin)s/pg_dump \\
--host=%(pgdata-directory)s \\ --host=%(pgdata-directory)s \\
--username postgres \\
--format=custom \\ --format=custom \\
--file=%(backup-directory)s/database.dump \\ --file=%(backup-directory)s/database.dump \\
%(dbname)s %(dbname)s
...@@ -102,6 +103,7 @@ class ImportRecipe(GenericBaseRecipe): ...@@ -102,6 +103,7 @@ class ImportRecipe(GenericBaseRecipe):
#!/bin/sh #!/bin/sh
%(bin)s/pg_restore \\ %(bin)s/pg_restore \\
--host=%(pgdata-directory)s \\ --host=%(pgdata-directory)s \\
--username postgres \\
--dbname=%(dbname)s \\ --dbname=%(dbname)s \\
--clean \\ --clean \\
--no-owner \\ --no-owner \\
......
...@@ -43,6 +43,25 @@ all slave instances. ...@@ -43,6 +43,25 @@ all slave instances.
Finally, the slave instance will be accessible from: Finally, the slave instance will be accessible from:
https://someidentifier.moulefrite.org. https://someidentifier.moulefrite.org.
How to have custom configuration in frontend server
===================================================
In your instance directory, you, as sysadmin, can directly edit two
configuration files that won't be overwritten by SlapOS to customize your
instance:
* $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.custom.conf
* $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.virtualhost.custom.conf
The first one is included in the end of the main apache configuration file.
The second one is included in the virtualhost of the main apache configuration file.
SlapOS will jsut create those two files for you, then completely forget them.
Note: make sure that the UNIX user of the instance has read access to those
files if you edit them.
Instance Parameters Instance Parameters
=================== ===================
......
...@@ -3,17 +3,22 @@ ...@@ -3,17 +3,22 @@
[maarch-configuration] [maarch-configuration]
recipe = slapos.recipe.maarch:default recipe = slapos.recipe.maarch:default
htdocs = $${apache-php:htdocs} htdocs = $${apache-php:htdocs}
db_host = $${postgres-urlparse:host} db-host = $${postgres-urlparse:host}
db_port = $${postgres-urlparse:port} db-port = $${postgres-urlparse:port}
db_dbname = $${postgres-urlparse:path} db-dbname = $${postgres-urlparse:path}
db_username = $${postgres-urlparse:username} db-username = $${postgres-urlparse:username}
db_password = $${postgres-urlparse:password} db-password = $${postgres-urlparse:password}
language = en language = en
php_ini = $${directory:php-ini-dir}/php.ini php-ini = $${directory:php-ini-dir}/php.ini
root_docservers = $${buildout:directory}/srv/docservers root-docservers = $${buildout:directory}/srv/docservers
dependency = $${apache-php:recipe} dependency = $${apache-php:recipe}
maarch-sql-data-file = $${slap-parameter:maarch-sql-data-file}
[publish-connection-informations] [publish-connection-informations]
# XXX login should not be hardcoded # XXX login should not be hardcoded
login = superadmin login = superadmin
password = $${maarch-configuration:db_password} password = $${maarch-configuration:db-password}
[slap-parameter]
maarch-sql-data-file =
...@@ -5,11 +5,6 @@ extends = ...@@ -5,11 +5,6 @@ extends =
../../stack/lapp/buildout.cfg ../../stack/lapp/buildout.cfg
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.recipe.maarch-repository
# += since we need rdiff-backup and friends # += since we need rdiff-backup and friends
parts += parts +=
apache-php-postgres apache-php-postgres
...@@ -17,27 +12,19 @@ parts += ...@@ -17,27 +12,19 @@ parts +=
eggs eggs
instance instance
instance-apache-php instance-apache-php
slapos.recipe.maarch-repository slapos-recipe-maarch-egg
check-recipe
[versions]
slapos.recipe.maarch =
[slapos.recipe.maarch-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.recipe.maarch.git
branch = master
git-executable = ${git:location}/bin/git
[check-recipe] #----------------
recipe = plone.recipe.command #--
stop-on-error = true #-- Explicitly provide the configuration egg,
update-command = ${:command} #-- otherwise instance.cfg won't be able to use it.
command = #--
grep parts ${buildout:develop-eggs-directory}/slapos.recipe.maarch.egg-link [slapos-recipe-maarch-egg]
recipe = zc.recipe.egg
eggs =
slapos.recipe.maarch
#---------------- #----------------
...@@ -63,7 +50,7 @@ part-list = maarch-configuration ...@@ -63,7 +50,7 @@ part-list = maarch-configuration
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-custom.cfg.in url = ${:_profile_base_location_}/instance-custom.cfg.in
output = ${buildout:directory}/instance-custom.cfg output = ${buildout:directory}/instance-custom.cfg
md5sum = d1b02a8c571886a225f2f1c8da06286c md5sum = f86b311d443156e327a1b5f5acfb22b1
mode = 0644 mode = 0644
...@@ -83,3 +70,6 @@ install_cmd = ...@@ -83,3 +70,6 @@ install_cmd =
#---------------- #----------------
[versions]
slapos.recipe.maarch = 0.4
...@@ -141,7 +141,7 @@ parts = ...@@ -141,7 +141,7 @@ parts =
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git repository = http://git.erp5.org/repos/slapos.git
branch = erp5-component branch = erp5
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[check-recipe] [check-recipe]
...@@ -346,7 +346,7 @@ repository_id_list = erp5 ...@@ -346,7 +346,7 @@ repository_id_list = erp5
[erp5] [erp5]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/erp5.git repository = http://git.erp5.org/repos/erp5.git
branch = erp5-component branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[fix-products-paths] [fix-products-paths]
......
...@@ -29,9 +29,13 @@ parts += ...@@ -29,9 +29,13 @@ parts +=
{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }} {{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
# Nothing to do for the exporter. Just dummy part that does nothing. # Nothing to do for the exporter. This wrapper is intended
# For httpd instance, PBS will directly pull data from srv/www. # to produce the "dump" files that have to be backed up.
# XXX-Cedric: write a real backup system. # So, in case of binary DB data, we would back up the ASCII dump
# files, or a similar data format that can be restored on any machine
# (i.e. postgres 'custom')
# In the case of an httpd instance, the src/www directory can be directly
# pulled from the PBS, we don't need to prepare anything.
[exporter] [exporter]
wrapper = /bin/true wrapper = /bin/true
......
[buildout]
parts =
urls
apache-proxy
logrotate
logrotate-entry-apache
cron
cron-entry-logrotate
sshkeys-authority
sshkeys-dropbear
dropbear-server
dropbear-server-pbs-authorized-key
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
#----------------
#--
#-- Creation of all needed directories.
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/service
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
ssh = $${rootdirectory:etc}/ssh
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
#----------------
#--
#-- sshkeys
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests
keys = $${directory:sshkeys}/keys
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
#----------------
#--
#-- Dropbear.
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
#----------------
#--
#-- rdiff
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
#----------------
#--
#-- Apache Proxy.
[apache-proxy]
recipe = slapos.cookbook:apacheproxy
url = $${slap-parameter:proxy-url}
pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6}
port = 8080
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
httpd-conf = $${rootdirectory:etc}/apache.conf
wrapper = $${basedirectory:services}/apache
promise = $${basedirectory:promises}/apache
httpd-binary = ${apache:location}/bin/httpd
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-proxy:error-log} $${apache-proxy:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
#----------------
#--
#-- Publish instance parameters.
[urls]
recipe = slapos.cookbook:publish
url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
# This file is responsible of three things:
# 1/ Act as "Apache exporter"
# 2/ Act as "Postgres backup infrastructure requester"
# 3/ Act as "Apache" instance
{% import 'parts' as parts %}
{% import 'replicated' as replicated %}
[buildout]
extends = {{templateapache}}
{{templatepbsreadyexport}}
parts +=
{{ parts.replicate("postgres", "3") }}
# Repeating parts from instance-apache-php.
# XXX-Cedric: how to simplify this?
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
promise
frontend-promise
content-promise
publish-connection-informations
{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
# Nothing to do for the exporter. This wrapper is intended
# to produce the "dump" files that have to be backed up.
# So, in case of binary DB data, we would back up the ASCII dump
# files, or a similar data format that can be restored on any machine
# (i.e. postgres 'custom')
# In the case of an httpd instance, the src/www directory can be directly
# pulled from the PBS, we don't need to prepare anything.
[exporter]
wrapper = /bin/true
# State that we want to backup srv/www directory, not srv/backup.
# XXX-Cedric: works well, but doesn't work with big data.
[rdiff-backup-server]
path = ${directory:www}
# Add "exporter" parameters to list of published connection parameters
[publish-connection-informations]
# XXX-Cedric: Long term goal: could be a recipe that requests an instance and
# bubbles ALL
# parameters of the requested instance. Requirement: aggregated publish.
<= resilient-publish-connection-parameter
# XXX-Cedric: resilient overwrites what's returned from request-postgres
# XXX-Cedric: change the request method to return everything from
# getConnectionParameterDict()
[request-postgres]
return = ssh-public-key ssh-url notification-id ip url
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
extends = ${pbsready-import:output}
[directory]
srv = $${buildout:directory}/srv
www = $${:srv}/www/
# Nothing to do for the import. Just dummy part that does nothing.
# For httpd instance, PBS will directly push data to srv/www.
# XXX-Cedric: write a real backup system.
[importer]
wrapper = /bin/true
# State that we want to push to srv/www directory, not srv/backup.
[rdiff-backup-server]
path = $${directory:www}
...@@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate ...@@ -57,6 +57,7 @@ logrotate-backup = $${basedirectory:backup}/logrotate
report = $${rootdirectory:etc}/report report = $${rootdirectory:etc}/report
stunnel-conf = $${rootdirectory:etc}/stunnel stunnel-conf = $${rootdirectory:etc}/stunnel
xml-report = $${rootdirectory:var}/xml_report xml-report = $${rootdirectory:var}/xml_report
www = $${rootdirectory:srv}/www/
[cadirectory] [cadirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -195,7 +196,7 @@ url = $${request-postgres:connection-url} ...@@ -195,7 +196,7 @@ url = $${request-postgres:connection-url}
recipe = slapos.cookbook:apachephp recipe = slapos.cookbook:apachephp
source = ${application:location} source = ${application:location}
htdocs = $${rootdirectory:srv}/www/ htdocs = $${directory:www}
pid-file = $${basedirectory:run}/apache.pid pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock lock-file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6} ip = $${slap-network-information:global-ipv6}
......
[buildout] [buildout]
ignore-existing = true
parts = parts =
slapos-cookbook slapos-cookbook
apache-php-postgres apache-php-postgres
...@@ -15,11 +18,14 @@ parts = ...@@ -15,11 +18,14 @@ parts =
instance-postgres-import instance-postgres-import
instance-postgres-export instance-postgres-export
#Contains the importer and exporter recipes for apache
instance-apache-import
instance-apache-export
extends = extends =
../resilient/buildout.cfg
../../component/apache/buildout.cfg
../../component/apache-php/buildout.cfg ../../component/apache-php/buildout.cfg
../../component/apache/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
...@@ -29,6 +35,7 @@ extends = ...@@ -29,6 +35,7 @@ extends =
../../component/rdiff-backup/buildout.cfg ../../component/rdiff-backup/buildout.cfg
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../resilient/buildout.cfg
../slapos.cfg ../slapos.cfg
...@@ -48,21 +55,35 @@ strip-top-level-dir = true ...@@ -48,21 +55,35 @@ strip-top-level-dir = true
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
md5sum = a482fa0e72839b4bd75b169ac1460d64 md5sum = 25d07b5101d5f566398686642ada4cee
mode = 0644 mode = 0644
[instance-apache-php] [instance-apache-php]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
output = ${buildout:directory}/instance-apache-php.cfg output = ${buildout:directory}/instance-apache-php.cfg
md5sum = 72b70452d1c077cfcd0f268181506b8e md5sum = 823257dda6f3068a38c6b69c771cf307
mode = 0644 mode = 0644
[instance-apache-backup] [instance-apache-import]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in url = ${:_profile_base_location_}/apache/instance-apache-import.cfg.in
output = ${buildout:directory}/instance-apache-backup.cfg output = ${buildout:directory}/instance-apache-import.cfg
md5sum = db879141c0b6a77ef8b3b7e699f5583a md5sum = f1dc2a71d362b5d2d36481ffefdd2293
mode = 0644
[instance-apache-export]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
rendered = ${buildout:directory}/instance-apache-export.cfg
context = key templateapache instance-apache-php:output
key templatepbsreadyexport pbsready-export:output
import-list = file parts template-parts:destination
file replicated template-replicated:destination
md5sum = bdc7e126567ece6bf93a9bb493e29fac
mode = 0644 mode = 0644
[instance-resilient] [instance-resilient]
...@@ -70,17 +91,16 @@ recipe = slapos.recipe.template:jinja2 ...@@ -70,17 +91,16 @@ recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
rendered = ${buildout:directory}/instance-resilient.cfg rendered = ${buildout:directory}/instance-resilient.cfg
context = key templateapache instance-apache-php:output context = key buildout buildout:bin-directory
key dropbear dropbear:location key develop_eggs_directory buildout:develop-eggs-directory
key buildout buildout:bin-directory key eggs_directory buildout:eggs-directory
import-list = file parts template-parts:destination import-list = file parts template-parts:destination
file replicated template-replicated:destination file replicated template-replicated:destination
md5sum = 46c7d8f691bd37d84e0bd03b83e51d14 md5sum = ef38aa9810ce20960382261f235abfcd
mode = 0644 mode = 0644
[instance-postgres] [instance-postgres]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in
...@@ -99,7 +119,7 @@ mode = 0644 ...@@ -99,7 +119,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in
output = ${buildout:directory}/instance-postgres-export.cfg output = ${buildout:directory}/instance-postgres-export.cfg
md5sum = 68080e5d861eb3474442211dd65c668b md5sum = d95205a5fc2825e9709ed6db295111e2
mode = 0644 mode = 0644
...@@ -155,7 +175,6 @@ eggs = ...@@ -155,7 +175,6 @@ eggs =
${psycopg2:egg} ${psycopg2:egg}
slapos.toolbox slapos.toolbox
[networkcache] [networkcache]
# Romain Courteaud + Sebastien Robin + Alain Takoudjou # Romain Courteaud + Sebastien Robin + Alain Takoudjou
# + Cedric de Saint Martin signature certificate # + Cedric de Saint Martin signature certificate
......
...@@ -4,172 +4,47 @@ ...@@ -4,172 +4,47 @@
{% import 'replicated' as replicated %} {% import 'replicated' as replicated %}
[buildout] [buildout]
eggs-directory = {{ eggs_directory }}
extends = develop-eggs-directory = {{ develop_eggs_directory }}
{{templateapache}} offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts += parts +=
{{ parts.replicate("postgres","3") }} {{ parts.replicate("apache", "3") }}
request-apache-backup-1
request-apache-backup-2
request-pull-backup-server-apache-1
request-pull-backup-server-apache-backup-1
request-pull-backup-server-apache-2
request-pull-backup-server-apache-backup-2
publish-connection-informations publish-connection-informations
apache-php
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
dropbear-server
sshkeys-authority
dropbear-server-pbs-authorized-key
request-pull-backup-server
{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
[request-pull-backup-server]
<= request-pbs-common
name = PBS (Pull Backup Server)
return = ssh-key notification-url feeds-url
slave = false
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = ${directory:sshkeys}/requests
keys = ${directory:sshkeys}/keys
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = ${sshkeys-directory:requests}
keys-directory = ${sshkeys-directory:keys}
wrapper = ${basedirectory:services}/sshkeys_authority
keygen-binary = {{dropbear}}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = ${dropbear-server:wrapper}
public-key = ${dropbear-server:rsa-keyfile}.pub
private-key = ${dropbear-server:rsa-keyfile}
wrapper = ${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = ${slap-network-information:global-ipv6}
port = 2222
home = ${directory:ssh}
wrapper = ${rootdirectory:bin}/raw_sshd
shell = ${rdiff-backup-server:wrapper}
rsa-keyfile = ${directory:ssh}/server_key.rsa
dropbear-binary = {{dropbear}}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = ${request-pull-backup-server:connection-ssh-key}
[rdiff-backup-server]
<= apache-php
recipe = slapos.cookbook:pbs
client = false
path = ${apache-php:htdocs}
wrapper = ${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = {{buildout}}/rdiff-backup
[request-apache-backup-1]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 1
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-apache-backup-2]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 2
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-pull-backup-server-apache-1]
<= request-pbs-common
name = PBS pulling from Apache 1
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-2]
<= request-pbs-common
name = PBS pulling from Apache 2
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-backup-1]
<= request-pbs-common
name = PBS pushing to ${request-apache-backup-1:name}
config = url name type server-key on-notification
config-url = ${request-apache-backup-1:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-1:config-name}
config-type = push
config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[request-pull-backup-server-apache-backup-2]
<= request-pbs-common
name = PBS pushing to ${request-apache-backup-2:name}
config = url name type server-key on-notification
config-url = ${request-apache-backup-2:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-2:config-name}
config-type = push
config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
slave = true
sla = instance_guid
sla-instance_guid = ${request-pull-backup-server:instance_guid}
[directory] {{ replicated.replicate("apache", "3", "apache-export", "apache-import") }}
ssh = ${rootdirectory:etc}/ssh
sshkeys = ${rootdirectory:srv}/sshkeys # Bubble up the parameters
[request-apache]
return = url ssh-public-key ssh-url notification-id ip url backend_url
# XXX: hardcoded values
config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid maarch-sql-data-file
config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid}
config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid}
config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid}
config-pbs-postgres2-computer-guid = ${slap-parameter:pbs-postgres2-computer-guid}
config-postgres3-computer-guid = ${slap-parameter:postgres3-computer-guid}
config-pbs-postgres3-computer-guid = ${slap-parameter:pbs-postgres3-computer-guid}
config-domain = ${slap-parameter:domain}
config-maarch-sql-data-file = ${slap-parameter:maarch-sql-data-file}
[publish-connection-informations]
recipe = slapos.cookbook:publish
backend_url = ${request-apache:connection-backend_url}
url = ${request-apache:connection-url}
[slap-parameter]
# Default parameters for distributed deployment
# I.e state "backup1 of postgres should go there, ..."
# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
postgres1-computer-guid =
pbs-postgres1-computer-guid =
postgres2-computer-guid =
pbs-postgres2-computer-guid =
postgres3-computer-guid =
pbs-postgres3-computer-guid =
# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
domain =
# the following parameters are specific to maarch
maarch-sql-data-file =
...@@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered} ...@@ -14,7 +14,8 @@ resilient = ${instance-resilient:rendered}
postgres = ${instance-postgres:output} postgres = ${instance-postgres:output}
postgres-import = ${instance-postgres-import:output} postgres-import = ${instance-postgres-import:output}
postgres-export = ${instance-postgres-export:output} postgres-export = ${instance-postgres-export:output}
apache-backup = ${instance-apache-backup:output} apache-import = ${instance-apache-import:output}
apache-export = ${instance-apache-export:rendered}
#frozen creates a syntax error, meaning it can keep its data. #frozen creates a syntax error, meaning it can keep its data.
#It's dirty as hell, it needs to be replaced. #It's dirty as hell, it needs to be replaced.
......
...@@ -19,3 +19,8 @@ bin = $${postgres-instance:bin} ...@@ -19,3 +19,8 @@ bin = $${postgres-instance:bin}
pgdata-directory = $${postgres-instance:pgdata-directory} pgdata-directory = $${postgres-instance:pgdata-directory}
backup-directory = $${postgres-instance:backup-directory} backup-directory = $${postgres-instance:backup-directory}
dbname = $${postgres-instance:dbname} dbname = $${postgres-instance:dbname}
# Extends publish section with resilient parameters
[urls]
<= resilient-publish-connection-parameter
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment