Skip to content

slapos
slapos
Commit 259aa9be authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
Browse files
Options

Merge remote-tracking branch 'origin/master' into erp5-component

parents 16a9ebce 42e60966
Hide whitespace changes
Inline Side-by-side
Showing with 277 additions and 49 deletions
setup.py
View file @ 259aa9be
......@@ -127,6 +127,7 @@ setup(name=name,
'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
'java = slapos.recipe.java:Recipe',
'jsondump = slapos.recipe.jsondump:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe',
'kvm = slapos.recipe.kvm:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
......@@ -180,6 +181,7 @@ setup(name=name,
'siptester = slapos.recipe.siptester:SipTesterRecipe',
'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
'slapcontainer = slapos.recipe.container:Recipe',
'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',
......
slapos/recipe/jsondump.py 0 → 100644
View file @ 259aa9be
from slapos.recipe.librecipe import GenericBaseRecipe
import json
import os
class Recipe(GenericBaseRecipe):
def install(self):
parameter_dict = {
key: value
for key, value in self.options.items()
if key not in ['json-output', 'recipe']
}
with os.fdopen(os.open(self.options['json-output'], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fout:
fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
return [self.options['json-output']]
update = install
slapos/recipe/slapconfiguration.py
View file @ 259aa9be
......@@ -24,6 +24,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import json
import os
import slapos.slap
from slapos.recipe.librecipe import unwrap
from ConfigParser import RawConfigParser
......@@ -87,6 +91,15 @@ class Recipe(object):
OPTCRE_match = RawConfigParser.OPTCRE.match
def __init__(self, buildout, name, options):
parameter_dict = self.fetch_parameter_dict(options)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
def fetch_parameter_dict(self, options):
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
......@@ -138,12 +151,7 @@ class Recipe(object):
options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
options['tap'] = tap_set
parameter_dict = self._expandParameterDict(options, parameter_dict)
match = self.OPTCRE_match
for key, value in parameter_dict.iteritems():
if match(key) is not None:
continue
options['configuration.' + key] = value
return self._expandParameterDict(options, parameter_dict)
def _expandParameterDict(self, options, parameter_dict):
options['configuration'] = parameter_dict
......@@ -158,3 +166,16 @@ class Serialised(Recipe):
return parameter_dict
else:
return {}
class JsonDump(Recipe):
def __init__(self, buildout, name, options):
parameter_dict = self.fetch_parameter_dict(options)
self._json_output = options['json-output']
with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
def install(self):
return [self._json_output]
update = install
software/apache-frontend/README.apache_frontend.txt
View file @ 259aa9be
......@@ -25,6 +25,7 @@ These parameters are :
* "-frontend-type" : the type to deploy frontends with. (default to 2)
* "-frontend-quantity" : The quantity of frontends to request (default to "default")
* "-frontend-i-state": The state of frontend i
* "-frontend-config-i-foo": Frontend i will be requested with parameter foo
* "-frontend-software-release-url": Software release to be used for frontends, default to the current software release
* "-sla-i-foo" : where "i" is the number of the concerned frontend (between 1 and "-frontend-quantity") and "foo" a sla parameter.
ex:
......@@ -38,6 +39,12 @@ will request the third frontend on COMP-1234. All frontends will be of software
Note: the way slaves are transformed to a parameter avoid modifying more than 3 lines in the frontend logic.
Important NOTE: The way you ask for slave to a replicate frontend is the same as the one you would use for the software given in "-frontend-quantity". Do not forget to use "replicate" for software type. XXXXX So far it is not possible to do a simple request on a replicate frontend if you do not know the software_guid or other sla-parameter of the master instance. In fact we do not know yet the software type of the "requested" frontends. TO BE IMPLEMENTED
XXX Should be moved to specific JSON File
Extra-parameter per frontend with default :
ram-cache-size = 1G
disk-cache-size = 8G
How to deploy a frontend server
===============================
......
software/apache-frontend/common.cfg
View file @ 259aa9be
......@@ -67,20 +67,20 @@ mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = 986988ed3a357f87fc023e14d87dcc5c
md5sum = 53de57ef78345cedd3c715a105539ca3
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum = 8ec70e6276daaed240faa5059514929c
md5sum = da22cc3b2095766c5e14b29afab2b760
mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum = 1c2a47e7e2aa58b4f350ed29ab1a98fe
md5sum = bae669cdc917c68186a387903478a53d
mode = 640
[template-slave-configuration]
......@@ -98,7 +98,7 @@ mode = 640
[template-apache-frontend-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache.conf.in
md5sum = 72922908c1f4e72c92bb03e072660c7c
md5sum = ce88924c53f09c9a3ef12ec4d8a8ad16
mode = 640
[template-apache-cached-configuration]
......@@ -160,7 +160,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[template-trafficserver-records-config]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
md5sum = 950a19be225a25309a3bda3f61fb5f6a
md5sum = a892d3e71988a8293e44382cbf10810f
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = records.config.jinja2
download-only = true
......
software/apache-frontend/instance-apache-frontend.cfg
View file @ 259aa9be
......@@ -118,8 +118,10 @@ configuration.apache_custom_https = ""
configuration.apache_custom_http = ""
configuration.apache-key =
configuration.apache-certificate =
configuration.apache-ca-certificate =
configuration.open-port = 80 443
configuration.extra_slave_instance_list =
configuration.disk-cache-size = 8G
[frontend-configuration]
template-log-access = ${template-log-access:target}
......@@ -255,6 +257,7 @@ extra-context =
key access_control_string apache-configuration:access-control-string
key login_certificate ca-frontend:cert-file
key login_key ca-frontend:key-file
key login_ca_crt ca-custom-frontend:rendered
key ca_dir certificate-authority:ca-dir
key ca_crl certificate-authority:ca-crl
key access_log apache-configuration:access-log
......@@ -379,6 +382,13 @@ cert-content = $${instance-parameter:configuration.apache-certificate}
# Put domain name
name = $${instance-parameter:configuration.domain}
[ca-custom-frontend]
< = jinja2-template-base
template = ${template-empty:target}
rendered = $${cadirectory:certs}/apache_frontend.ca.crt
extra-context =
key content instance-parameter:configuration.apache-ca-certificate
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
......@@ -456,7 +466,7 @@ local-ip = $${instance-parameter:ipv4-random}
input-port = 23432
hostname = $${slap-parameter:frontend-name}
remap = map / http://$${instance-parameter:ipv4-random}:$${apache-configuration:cache-through-port}
disk-cache-config = $${trafficserver-directory:cache-path} 8G volume=$${slap-parameter:frontend-name}
disk-cache-config = $${trafficserver-directory:cache-path} $${instance-parameter:configuration.disk-cache-size} volume=$${slap-parameter:frontend-name}
plugin-config = ${trafficserver:location}/libexec/trafficserver/rfc5861.so
[trafficserver-configuration-directory]
......@@ -481,7 +491,7 @@ environment = TS_ROOT=$${buildout:directory}
template = ${template-trafficserver-records-config:location}/${template-trafficserver-records-config:filename}
rendered = $${trafficserver-directory:configuration}/records.config
mode = 700
context =
extra-context =
import os_module os
section ats_directory trafficserver-directory
section ats_configuration trafficserver-variable
......
software/apache-frontend/instance-apache-replicate.cfg.in
View file @ 259aa9be
......@@ -23,6 +23,7 @@ context =
{% set slave_list_name = 'extra_slave_instance_list' -%}
{% set frontend_list = [] %}
{% set frontend_section_list = [] %}
{% set request_dict = {} %}
{% set namebase = 'apache-frontend' -%}
# XXX Dirty hack, not possible to define default value before
{% set sla_computer_apache_1_key = '-sla-1-computer_guid' -%}
......@@ -37,28 +38,30 @@ context =
{% set request_section_title = 'request-%s' % frontend_name -%}
{% set sla_key = "-sla-%s-" % i -%}
{% set sla_key_length = sla_key | length %}
{% set sla_parameters = [] %}
{% set sla_dict = {} %}
{% set config_key = "-frontend-config-%s-" % i %}
{% set config_key_length = config_key | length %}
{% set config_dict = {} %}
{% for key in slapparameter_dict.keys() %}
{% if key.startswith(sla_key) %}
{% do sla_parameters.append(key[sla_key_length:]) %}
{% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
# We check for specific configuration regarding the frontend
{% elif key.startswith(config_key) -%}
{% do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %}
{% endif -%}
{% endfor -%}
{% do frontend_list.append(frontend_name) -%}
{% do frontend_section_list.append(request_section_title) -%}
{% do part_list.append(request_section_title) -%}
[{{request_section_title}}]
<= replicate
name = {{frontend_name}}
# Filling request dict for slave
{% set state_key = "-frontend-%s-state" % i %}
{% if slapparameter_dict.has_key(state_key) %}
state = {{ slapparameter_dict.pop(state_key) }}
{% endif%}
{% if sla_parameters %}
sla = {{ ' '.join(sla_parameters) }}
{% for parameter in sla_parameters -%}
sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
{% endfor -%}
{% endif -%}
{% do request_dict.__setitem__(request_section_title,
{
'config': config_dict,
'name': frontend_name,
'sla': sla_dict,
'state': slapparameter_dict.pop(state_key, None)
}) %}
{% endfor -%}
{% set authorized_slave_string = slapparameter_dict.pop('-frontend-authorized-slave-string', '') -%}
......@@ -83,13 +86,29 @@ software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
config = {{ ' '.join(slapparameter_dict.keys()) + ' ' + slave_list_name }}
{% for parameter, value in slapparameter_dict.iteritems() -%}
config-{{parameter}} = {{ value }}
{% endfor -%}
config-{{ slave_list_name }} = {{ json_module.dumps(authorized_slave_list) }}
config = _
connection-monitor_url =
{% for section, frontend_request in request_dict.iteritems() %}
[{{section}}]
<= replicate
name = {{ frontend_request.get('name') }}
{% if frontend_request.get('state') %}
state = {{ frontend_request.get('state') }}
{% endif%}
{% set slave_configuration_dict = frontend_request.get('config') %}
{% do slave_configuration_dict.update(**slapparameter_dict) %}
{% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
config-_ = {{ json_module.dumps(slave_configuration_dict) }}
{% if frontend_request.get('sla') %}
sla = {{ ' '.join(frontend_request.get('sla').keys()) }}
{% for parameter, value in frontend_request.get('sla').iteritems() -%}
sla-{{ parameter }} = {{ value }}
{% endfor -%}
{% endif -%}
{% endfor -%}
[publish-information]
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
......
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 259aa9be
......@@ -189,7 +189,7 @@ extra-context =
raw http_port {{ http_port }}
{{ '\n' }}
{% do slave_publish_dict.update(**{'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('custom_domain'), 'url':"http://%s" % slave_instance.get('custom_domain'), 'site_url':"http://%s" % slave_instance.get('custom_domain')}) %}
{% do slave_publish_dict.update(**{'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('custom_domain'), 'url':"http://%s" % slave_instance.get('custom_domain'), 'site_url':"http://%s" % slave_instance.get('custom_domain'), 'secure_access': 'https://%s' % slave_instance.get('custom_domain')}) %}
{% endif -%}
......
software/apache-frontend/templates/apache.conf.in
View file @ 259aa9be
......@@ -97,14 +97,6 @@ BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
IPReadLimit {{ slapparameter_dict.get('ip-read-limit', '10') }}
</IfModule>
# Cache directives
CacheEnable mem /
CacheDefaultExpire 3600
MCacheSize 8192
MCacheMaxObjectCount 1000
MCacheMaxObjectSize 8192
MCacheRemovalAlgorithm LRU
# Deflate
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
BrowserMatch ^Mozilla/4 gzip-only-text/html
......@@ -114,6 +106,9 @@ BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
# SSL Configuration
SSLCertificateFile {{ login_certificate }}
SSLCertificateKeyFile {{ login_key }}
{% if slapparameter_dict.get('apache-ca-certificate') %}
SSLCACertificateFile {{ login_ca_crt }}
{% endif %}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)
......
software/apache-frontend/templates/trafficserver/records.config.jinja2
View file @ 259aa9be
......@@ -228,8 +228,8 @@ CONFIG proxy.config.http.cache.http INT 1
# Enabling this setting allows the proxy to cache empty documents. This currently
# requires that the response has a Content-Length: header, with a value of "0".
CONFIG proxy.config.http.cache.allow_empty_doc INT 0
CONFIG proxy.config.http.cache.ignore_client_no_cache INT 1
CONFIG proxy.config.http.cache.ims_on_client_no_cache INT 1
CONFIG proxy.config.http.cache.ignore_client_no_cache INT 0
CONFIG proxy.config.http.cache.ims_on_client_no_cache INT 0
CONFIG proxy.config.http.cache.ignore_server_no_cache INT 0
CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 0
CONFIG proxy.config.http.normalize_ae_gzip INT 0
......@@ -331,8 +331,7 @@ CONFIG proxy.config.cache.permit.pinning INT 0
# default the ram cache size to AUTO_SIZE (-1) based on cache size
# (approximately 10 MB of RAM cache per GB of disk cache)
# alternatively, set to a fixed value such as 21474836480 (20GB)
CONFIG proxy.config.cache.ram_cache.size INT 1G
CONFIG proxy.config.cache.ram_cache_cutoff INT 4194304
CONFIG proxy.config.cache.ram_cache.size INT {{ slapparameter_dict.get('ram-cache-size', '1G') }}
# Replacement algorithm
# 0 : Clocked Least Frequently Used by Size (CLFUS) w/optional compression
# 1 : LRU w/o optional compression - trivially simple
......@@ -366,9 +365,6 @@ CONFIG proxy.config.cache.target_fragment_size INT 1048576
CONFIG proxy.config.cache.max_doc_size INT 0
# enable the cache to read from an object while it is being added to the cache
CONFIG proxy.config.cache.enable_read_while_writer INT 0
# This controls how many objects (average) the disk caches can hold, and
# how much memory it'll consume for the directory structure.
CONFIG proxy.config.cache.min_average_object_size INT 8000
# How many I/O threads to allocate per disk (spindle). Be aware that RAID
# disks would show up to TS as a single spindle.
CONFIG proxy.config.cache.threads_per_disk INT 8
......
software/test-slave-instance-deployment/instance-default.cfg
View file @ 259aa9be
......@@ -2,10 +2,19 @@
parts =
publish-connection-informations
request-slave-instance
check-connection-informations-promise
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directories]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc
scripts = $${:etc}/run
services = $${:etc}/service
promises = $${:etc}/promise
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
......@@ -43,3 +52,16 @@ test-instance-state = $${request-test-instance:instance-state}
test-instance-slave-instance-list = $${request-test-instance:connection-slave-instance-list}
root-instance-slave-instance-list = $${slap-configuration:slave-instance-list}
slave-instance-parameter = $${request-slave-instance:connection-foo}
# Create a file containing connection parameters
# Then create a promise that will check content of this file
[dump-connection-informations]
<= publish-connection-informations
recipe = slapos.cookbook:jsondump
json-output = $${buildout:directory}/connection-parameters.json
[check-connection-informations-promise]
recipe = collective.recipe.template
input = ${promise-template:output}
output = $${directories:promises}/checkparameters
mode = 700
software/test-slave-instance-deployment/parameter-check-promise.py.in 0 → 100644
View file @ 259aa9be
#!${buildout:executable}
import sys
import os
import json
json_file = '$${dump-connection-informations:json-output}'
connection_parameter_list = json.loads(open(json_file, 'r').read())
# XXX list all problems before exit
# XXX: factor in a clean dict loop
test_instance_guid = connection_parameter_list['test-instance-guid']
if not test_instance_guid == 'slappart1':
print 'Bad test-instance-guid connection parameter: %s' % test_instance_guid
sys.exit(1)
test_instance_state = connection_parameter_list['test-instance-state']
if not test_instance_state == 'started':
print 'Bad test-instance-state connection parameter: %s instead of started' % test_instance_state
sys.exit(1)
test_instance_slave_instance_list = connection_parameter_list['test-instance-slave-instance-list']
if test_instance_slave_instance_list == []:
print 'test-instance-slave-instance-list connection parameter is empty.'
sys.exit(1)
root_instance_slave_instance_list = connection_parameter_list['root-instance-slave-instance-list']
if root_instance_slave_instance_list != []:
print 'root-instance-slave-instance-list connection parameter is not empty.'
sys.exit(1)
slave_instance_parameter = connection_parameter_list['slave-instance-parameter']
if slave_instance_parameter != 'I am slave instance, and I am deployed!':
print 'parameter slave-instance-parameter is not correct.'
sys.exit(1)
software/test-slave-instance-deployment/software.cfg
View file @ 259aa9be
# This Software Release is used to test
# the behavior of Slave Instances in slapos and in slaprunner.
[buildout]
extends = ../../stack/slapos.cfg
develop =
/srv/slapgrid/slappart74/srv//runner/project/slapos
parts =
slapos-cookbook
instance-profile
collective-egg
[instance-profile]
recipe = slapos.recipe.template
......@@ -24,6 +28,16 @@ url = ${:_profile_base_location_}/instance-test.cfg
output = ${buildout:directory}/instance-test.cfg
mode = 0644
[promise-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/parameter-check-promise.py.in
output = ${buildout:directory}/parameter-check-promise.py
mode = 0644
[collective-egg]
recipe = zc.recipe.egg
eggs = collective.recipe.template
[versions]
Jinja2 = 2.7.3
MarkupSafe = 0.23
......@@ -52,6 +66,7 @@ slapos.recipe.template = 2.5
stevedore = 0.15
supervisor = 3.1.0
xml-marshaller = 0.9.7
collective.recipe.template = 1.11
# Required by:
# slapos.core==1.1.2
......@@ -76,3 +91,93 @@ pycparser = 2.10
# Required by:
# slapos.core==1.1.2
zope.interface = 4.1.1
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout]
# Explicitly disable download-cache
download-cache =
# Generate list of automatically chosen eggs version
extensions +=
buildout-versions
mr.developer
# Use shacache and lxml
extends =
../../component/lxml-python/buildout.cfg
# Separate from site eggs
allowed-eggs-from-site-packages =
include-site-packages = false
exec-sitecustomize = false
# Add location for modified non-official slapos.buildout
find-links +=
http://www.nexedi.org/static/packages/source/slapos.buildout/
http://www.nexedi.org/static/packages/source/
http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
# Use only quite well working sites.
allow-hosts +=
*.googlecode.com
*.nexedi.org
*.python.org
*.sourceforge.net
alastairs-place.net
bitbucket.org
dist.repoze.org
effbot.org
github.com
launchpad.net
peak.telecommunity.com
sourceforge.net
www.dabeaz.com
www.owlfish.com
# XXX: Workaround of SlapOS limitation
# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
# development / fast switching environment for whole software
unzip = true
# Define where is defined what version of eggs we should use
versions = versions
# Define networkcache with shacache.org
networkcache-section = networkcache
# Install slapos.cookbook containing all officials recipes
# Explicitely define dependencies as well, because of buildout limitation
# if using "develop"
[slapos-cookbook]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.cookbook
cliff
hexagonit.recipe.download
inotifyx
netaddr
netifaces
requests
slapos.core
supervisor
xml_marshaller
pytz
[versions]
# Use SlapOS patched zc.buildout
zc.buildout = 1.6.0-dev-SlapOS-012
# zc.recipe.egg 2.x is for Buildout 2
zc.recipe.egg = 1.3.2
# Use own version of h.r.download to be able to open xz-like archives
hexagonit.recipe.download = 1.7nxd002
# Use pinned version of setuptools. Other versions work, but changing
# version makes buildout recompile everything. Developers' nightmare.
setuptools = 0.9.8
# Official egg of prettytable has permission problems in EGG-INFO.
prettytable = 0.7.3-nxd001
[networkcache]
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7