Commit c7d00913 authored by Vincent Pelletier's avatar Vincent Pelletier

Initial neoppod commit.

parent eab8fdf7
......@@ -103,6 +103,9 @@ setup(name=name,
'mydumper = slapos.recipe.mydumper:Recipe',
'mysql = slapos.recipe.mysql:Recipe',
'nbdserver = slapos.recipe.nbdserver:Recipe',
'neoppod.admin = slapos.recipe.neoppod:Admin',
'neoppod.master = slapos.recipe.neoppod:Master',
'neoppod.storage = slapos.recipe.neoppod:Storage',
'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
'notifier.callback = slapos.recipe.notifier:Callback',
'notifier.notify = slapos.recipe.notifier:Notify',
......
from slapos.recipe.librecipe import GenericBaseRecipe
class NeoBaseRecipe(GenericBaseRecipe):
_binding_port_mandatory = True
def install(self):
options = self.options
if not options['masters']:
# All parameters are always provided.
# This parameter needs special care, because it is initially generated
# empty, until all requested master nodes get their partitions
# allocated.
# Only then can this recipe start succeeding and actually doing anything
# useful, as per NEO deploying constraints.
raise Exception('"masters" parameter is mandatory')
option_list = [
options['binary'],
'-l', options['logfile'],
'-m', options['masters'],
'-b', self._getBindingAddress(),
# TODO: reuse partition reference for better log readability.
#'-n', options['name'],
'-c', options['cluster'],
]
if options['verbose']:
option_list.append('-v')
option_list.extend(self._getOptionList())
return [self.createPythonScript(
options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
option_list
)]
def _getBindingAddress(self):
options = self.options
bind = options['ip']
if 'port' in options:
# Some node types support port auto-allocation when no binding port is
# requested.
bind = bind + ':' + options['port']
elif self._binding_port_mandatory:
raise ValueError('"port" option is mandatory.')
return bind
def _getOptionList(self):
raise NotImplementedError
class Storage(NeoBaseRecipe):
_binding_port_mandatory = False
def _getOptionList(self):
return [
'-d', self.options['database-parameters'],
'-a', self.options['database-adapter'],
'-w', self.options['wait-database'],
]
class Admin(NeoBaseRecipe):
def _getOptionList(self):
return []
class Master(NeoBaseRecipe):
def _getOptionList(self):
options = self.options
return [
'-p', options['partitions'],
'-r', options['replicas'],
]
Software types
==============
Which software type is an entry point and can be used for root software
instance.
Parameters are expected to be passed as of *.serialised recipes expect them.
Minimal parameters::
```
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="_">{
'cluster': 'dummy',
}</parameter>
</instance>
```
For each available key in the outmost dict are described below.
default (default)
-----------------
Deploy a NEO cluster.
'cluster' (str, mandatory)
~~~~~~~~~~~~~~~~~~~~~~~~~~
Cluster unique identifier. Your last line of defense against mixing up neo
clusters and corrupting your data. Choose a unique value for each of your
cluster.
'partitions' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of partitions. You cannot change this value once you created a cluster.
Defautls to 12.
'replicas' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of replicates.
Defaults to 0 (no resilience).
'verbose' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~
Non-zero to increase verbosity.
Defaults to 0.
'mysql-storage-count' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of MySQL-based storage nodes to deploy. One master node is deployed
along with each storage.
Defaults to 1.
'admin-count' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of admin nodes to deploy.
Defaults to 1.
{% if slap_software_type not in (
mysql_storage_software_type,
admin_software_type,
) -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%}
{% set master_list = [] -%}
{% set admin_list = [] -%}
[request-common]
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
config-cluster = {{ slapparameter_dict['cluster'] }}
config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
[request-config]
base = cluster verbose
master = partitions replicas
node = masters
[node-base]
< = request-common
config = ${request-config:base} ${request-config:node}
config-masters = ${all-masters:addresses}
[mysql-storage-base]
< = request-common
config = ${request-config:base} ${request-config:master}
software-type = {{ mysql_storage_software_type }}
[mysql-storage-request-base]
< = mysql-storage-base
return = master
[admin-base]
< = node-base
return = admin
software-type = {{ admin_software_type }}
[publish]
recipe = slapos.cookbook:publish
masters = ${all-masters:addresses}
admins = ${all-admins:addresses}
{% for node_number in range(slapparameter_dict.get('mysql-storage-count', 1)) -%}
{% set section_id = 'storage-%i' % (node_number, ) -%}
{% set final_section_id = 'final-' ~ section_id -%}
{% do master_list.append(section_id) -%}
[{{ section_id }}]
< = mysql-storage-request-base
name = {{ section_id }}
[{{ section(final_section_id) }}]
< = mysql-storage-base
node-base
name = {{ section_id }}
config = ${request-config:base} ${request-config:master} ${request-config:node}
{% endfor -%}
{% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}
{% set section_id = 'admin-%i' % (node_number, ) -%}
{% do admin_list.append(section_id)%}
[{{ section(section_id) }}]
< = admin-base
name = {{ section_id }}
{% endfor -%}
[buildout]
parts =
{{ part_list | join('\n\t') }}
publish
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[all-masters]
addresses = {% for master in master_list -%}
{{ '${' + master + ':connection-master}' -}}
{% if not loop.last %} {% endif -%}
{% endfor %}
[all-admins]
addresses = {% for admin in admin_list -%}
{{ '${' + admin + ':connection-admin}' -}}
{% if not loop.last %} {% endif -%}
{% endfor %}
{% endif -%}
[buildout]
parts =
neo-admin-run-wrapper
neo-admin-promise
eggs-directory = {{eggs_directory}}
develop-eggs-directory = {{develop_eggs_directory}}
offline = true
[neo-admin]
recipe = slapos.cookbook:neoppod.admin
binary = {{bin_directory}}/neoadmin
wrapper = ${directory:bin}/neoadmin
logfile = ${directory:log}/neoadmin.log
ip = ${publish:ip}
port = ${publish:port}
cluster = ${slap-parameter:cluster}
masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
[publish]
recipe = slapos.cookbook:publish
# TODO: make port a partition parameter
# TODO: stop using slap-network-information
ip = [${slap-network-information:global-ipv6}]
port = 10002
admin = ${:ip}:${:port}
[neo-admin-run-wrapper]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:run}
link-binary = ${neo-admin:wrapper}
[directory]
recipe = slapos.cookbook:mkdirectory
run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log
bin = ${buildout:directory}/bin
[neo-admin-promise]
recipe = slapos.cookbook:check_port_listening
hostname = ${neo-admin:ip}
port = ${neo-admin:port}
path = ${directory:promises}/neo-admin-promise
[buildout]
parts =
neo-master-run-wrapper
neo-master-promise
eggs-directory = {{eggs_directory}}
develop-eggs-directory = {{develop_eggs_directory}}
offline = true
[neo-master]
recipe = slapos.cookbook:neoppod.master
binary = {{bin_directory}}/neomaster
wrapper = ${directory:bin}/neomaster
logfile = ${directory:log}/neomaster.log
ip = ${publish:ip}
port = ${publish:port}
cluster = ${slap-parameter:cluster}
partitions = ${slap-parameter:partitions}
replicas = ${slap-parameter:replicas}
masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
[publish]
recipe = slapos.cookbook:publish
# TODO: make port a partition parameter
# TODO: stop using slap-network-information
ip = [${slap-network-information:global-ipv6}]
port = 10000
master = ${:ip}:${:port}
[neo-master-run-wrapper]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:run}
link-binary = ${neo-master:wrapper}
[directory]
recipe = slapos.cookbook:mkdirectory
run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log
bin = ${buildout:directory}/bin
[neo-master-promise]
recipe = slapos.cookbook:check_port_listening
hostname = ${neo-master:ip}
port = ${neo-master:port}
path = ${directory:promises}/neo-master-promise
[slap-parameter]
# "masters" parameter is not provided when just requesting a partition.
# No actuall installation takes place at that time
# (slapos.cookbook:neoppod.master raises), but cfg expansion must succeed. So
# this default value is required.
masters =
[buildout]
extends = {{ master_cfg }}
parts += neo-storage-run-wrapper
[mariadb-instance]
recipe = slapos.cookbook:generic.mysql
user = user
database = neo
conf-file = ${directory:etc}/mariadb.cnf
# No networking (ip & port options)
data-directory = ${directory:srv_mariadb}
pid-file = ${directory:var_run}/mariadb.pid
socket = ${directory:var_run}/mariadb.sock
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
wrapper = ${directory:etc_run}/mariadb
update-wrapper = ${directory:etc_run}/mariadb_update
mysql-base-directory = {{ mariadb_location }}
mysql-binary = {{ mariadb_location }}/bin/mysql
mysql-install-binary = {{ mariadb_location }}/scripts/mysql_install_db
mysql-upgrade-binary = {{ mariadb_location }}/bin/mysql_upgrade
mysqld-binary = {{ mariadb_location }}/bin/mysqld
mroonga =
[neo-storage]
recipe = slapos.cookbook:neoppod.storage
binary = {{ bin_directory }}/neostorage
wrapper = ${directory:bin}/neostorage
logfile = ${directory:log}/neostorage.log
ip = [${slap-network-information:global-ipv6}]
cluster = ${slap-parameter:cluster}
masters = ${slap-parameter:masters}
verbose = ${slap-parameter:verbose}
database-adapter = MySQL
database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket}
wait-database = 60
[neo-storage-run-wrapper]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:etc_run}
link-binary = ${neo-storage:wrapper}
[directory]
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
etc_run = ${:etc}/run
var_run = ${:var}/run
srv_mariadb = ${buildout:directory}/srv/mariadb
{% set admin_software_type = 'neo-admin' -%}
{% set mysql_storage_software_type = 'neo-storage-mysql' -%}
[buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-connection]
computer-id = ${slap_connection:computer_id}
partition-id = ${slap_connection:partition_id}
server-url = ${slap_connection:server_url}
software-release-url = ${slap_connection:software_release_url}
key-file = ${slap_connection:key_file}
cert-file = ${slap_connection:cert_file}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[neo-cluster]
recipe = slapos.recipe.template:jinja2
template = {{ cluster }}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/cluster.cfg
extensions = jinja2.ext.do
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type slap-configuration:slap-software-type
key slapparameter_dict slap-configuration:configuration
raw admin_software_type {{ admin_software_type }}
raw mysql_storage_software_type {{ mysql_storage_software_type }}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${neo-cluster:rendered}
{{ admin_software_type }} = {{ neo_admin }}
{{ mysql_storage_software_type }} = {{ neo_storage_mysql }}
# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be
# dropped when a cookbook version containing neoppod recipe is released as an
# egg on pypi.
# Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
# a slapos.cookbook dependency, so it should be fetched automatically. But when
# automatically fetched, it gets built against system headers/libs, which is
# forbidden in slapos. So we need to fetch lxml explicitly so it is properly
# built.
[buildout]
extends =
../../stack/slapos.cfg
#COOKBOOK
../../component/git/buildout.cfg
#END COOKBOOK
#LXML
../../component/lxml-python/buildout.cfg
#END LXML
../../component/python-2.7/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg
develop =
#COOKBOOK
# Note: as we develop a path we fetch, first buildout execution will fail as
# develop couldn't find the clone. Second (and later) runs will succeed.
${:parts-directory}/slapos.cookbook-repository
#END COOKBOOK
parts =
#COOKBOOK
slapos.cookbook-repository
check-recipe
#END COOKBOOK
slapos-deps-eggs
# NEO & dependencies
python2.7
mariadb
mysql-python
neoppod
# NEO instanciation
template-instance
[slapos-deps-eggs]
recipe = zc.recipe.egg
eggs =
#LXML
${lxml-python:egg}
#END LXML
#COOKBOOK
slapos.cookbook
#END COOKBOOK
#LXML
[lxml-python]
python = python2.7
#END LXML
#COOKBOOK
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
branch = neoppod
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
#END COOKBOOK
[base-template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key bin_directory buildout:bin-directory
${:extra-context}
[template-neo-master]
< = base-template
filename = instance-neo-master.cfg
md5sum = b48e9446fb8fdce24d954d7d4f4c7460
[template-neo-storage-mysql]
< = base-template
filename = instance-neo-storage-mysql.cfg
md5sum = e767c66e7de58232de30bdf26188efe1
extra-context =
key mariadb_location mariadb:location
key master_cfg template-neo-master:rendered
[template-neo-admin]
< = base-template
filename = instance-neo-admin.cfg
md5sum = fcf1a5c937a609a934c8ccd7c1326726
[template-cluster]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/cluster.cfg.in
md5sum = fe37b2ff36fee65b44945393f63e51c3
[template-instance]
< = base-template
template = ${:_profile_base_location_}/instance.cfg.in
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
md5sum = 17e761b371487b55b39da1c39ebb1bb6
extra-context =
key neo_master template-neo-master:rendered
key neo_storage_mysql template-neo-storage-mysql:rendered
key neo_admin template-neo-admin:rendered
key cluster template-cluster:target
[neoppod]
recipe = zc.recipe.egg
eggs = neoppod[admin, master, storage-mysqldb]
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment