Commit 4a9cb977 authored by Cédric Le Ninivin's avatar Cédric Le Ninivin Committed by Kazuhiko Shiozaki

apache-frontend: Publish url per frontend per slave to access logs

apache-frontend: log access, one per slave

apache-frontend: protect log access with password

apache-frontend: slave publish their log-access-url

apache-frontend: publish all log-access urls

apache-frontend: comment md5 for development

apache-frontend: slapos.recipe.cmmi updated

apache-frontend: fix log access

apache-frontend: specify frontend for the log access

apache-frontend: update md5sum

apache-frontend: update versions
parent 15392af4
......@@ -36,6 +36,7 @@ recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.toolbox
plone.recipe.command
scripts =
killpidfromfile
......@@ -58,7 +59,7 @@ mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = 9f3eec91f43ae0730e9bba93f83572fc
md5sum = f0a507fed2b1dcab5530c892adce8327
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
......@@ -71,7 +72,7 @@ mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum = f5eef006211809669b12422240c6f436
md5sum = f002a8fc8fc5d18adbd8ac1ee054e852
mode = 640
[template-slave-configuration]
......@@ -83,13 +84,13 @@ mode = 640
[template-replicate-publish-slave-information]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/replicate-publish-slave-information.cfg.in
md5sum = 61a14dff06718e3d90c346a0a7b20c5a
md5sum = a2cf00b24877bf747e2408d444f16f05
mode = 640
[template-apache-frontend-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache.conf.in
md5sum = b4d7461c29fb6c36f09e48fa5ad59fba
md5sum = bae89ebc6c5e75b12535fbd6c37f647d
mode = 640
[template-apache-cached-configuration]
......@@ -129,6 +130,12 @@ url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
md5sum = ac845c0fa3835832307a0e7323cb339d
mode = 640
[template-log-access]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/template-log-access.conf.in
md5sum = f85005b430978f3bd24ee7ce11b0e304
mode = 640
[template-squid-configuration]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/squid.conf.jinja2
......
......@@ -27,6 +27,7 @@ parts =
promise-apache-frontend-v6-http
promise-apache-cached
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
......@@ -87,6 +88,13 @@ configuration.apache-certificate =
configuration.open-port = 80 443
configuration.extra_slave_instance_list =
[frontend-configuration]
template-log-access = ${template-log-access:target}
log-access-configuration = $${directory:etc}/apache-log-access.conf
apache-directory = ${apache-2.2:location}
apache-ipv6 = $${instance-parameter:ipv6-random}
apache-https-port = $${instance-parameter:configuration.port}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
......@@ -129,6 +137,9 @@ extra-context =
raw template_slave_configuration ${template-slave-configuration:target}
raw template_rewrite_cached ${template-rewrite-cached:target}
raw software_type single-custom-personal
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
[dynamic-custom-group-template-slave-list]
< = jinja2-template-base
......@@ -214,6 +225,7 @@ extra-context =
key error_log apache-configuration:error-log
key pid_file apache-configuration:pid-file
key slave_configuration_directory apache-directory:slave-configuration
section frontend_configuration frontend-configuration
[apache-frontend]
recipe = slapos.cookbook:wrapper
......@@ -366,7 +378,7 @@ state-file = $${directory:srv}/logrotate.status
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-directory:slave-log}/*_log $${apache-configuration:error-log} $${apache-configuration:access-log}
log = $${apache-configuration:error-log} $${apache-configuration:access-log}
frequency = daily
rotatep-num = 30
post = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:pid-file} SIGUSR1
......@@ -425,7 +437,7 @@ pid-filename-path = $${directory:run}/squid.pid
template = ${template-squid-configuration:target}
rendered = $${squid-cache:configuration-path}
extra-context =
key ip squid-cache:ip
key ip squid-cache:ip
key port squid-cache:port
key backend_ip squid-cache:backend-ip
key backend_port squid-cache:backend-port
......
......@@ -13,7 +13,7 @@ meld3 = 0.6.10
pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1
slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.34.0
smmap = 0.8.2
......@@ -23,7 +23,15 @@ cmd2 = 0.6.5.1
prettytable = 0.7.2
requests = 1.2.3
slapos.cookbook = 0.82
cffi = 0.8.2
cryptography = 0.3
plone.recipe.command = 1.1
pyOpenSSL = 0.14
six = 1.6.1
# Required by:
# cffi==0.8.2
pycparser = 2.10
# Required by:
# slapos.cookbook==0.82
lock-file = 2.0
......
......@@ -4,8 +4,9 @@
{% set part_list = [] -%}
{% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
{% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
{% set slave_log_dict = {} -%}
{% if extra_slave_instance_list -%}
{% set slave_instance_information_list = []-%}
{% set slave_instance_information_list = [] -%}
{% set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
{% endif -%}
[jinja2-template-base]
......@@ -17,15 +18,73 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
${:extra-context}
{% do logrotate_dict.pop('recipe') %}
[logrotate]
{% for key, value in logrotate_dict.iteritems() -%}
{{ key }} = {{ value }}
{% endfor %}
# Loop trhought slave list to set up slaves
{% for slave_instance in slave_instance_list -%}
{% set slave_reference = slave_instance.get('slave_reference') -%}
{% set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference -%}
{% set slave_parameter_dict = generic_instance_parameter_dict.copy() -%}
{% do part_list.append(slave_section_title) -%}
{% set slave_directory_section = slave_reference + "-directory" -%}
{% set slave_log_folder = logrotate_dict.get('backup') + '/' + slave_reference + "-logs" -%}
[{{slave_directory_section}}]
recipe = slapos.cookbook:mkdirectory
log-folder = {{slave_log_folder}}
# Set Up log files
{% do slave_parameter_dict.__setitem__('access_log', '/'.join([apache_log_directory, '%s_access_log' % slave_reference])) -%}
{% do slave_parameter_dict.__setitem__('error_log', '/'.join([apache_log_directory, '%s_error_log' % slave_reference])) -%}
# Set slave logrotate entry
{% set slave_logrotate_section = slave_reference + "-logs" -%}
{% do part_list.append(slave_logrotate_section) -%}
[{{slave_logrotate_section}}]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = ${:_buildout_section_name_}
log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
backup = {{ '${' + slave_directory_section + ':log-folder}' }}
frequency = daily
rotatep-num = 30
post = ${buildout:bin-directory}/killpidfromfile {{ apache_configuration.get('pid-file') }} SIGUSR1
sharedscripts = true
notifempty = true
create = true
# integrate current logs inside
{% set slave_ln_section = slave_reference + "-ln" -%}
{% do part_list.append(slave_ln_section) -%}
[{{slave_ln_section}}]
recipe = plone.recipe.command
stop-on-error = false
command = ln -s {{slave_parameter_dict.get('error_log')}} {{ '${' + slave_directory_section + ':log-folder}' }}/apache-error.log && ln -s {{slave_parameter_dict.get('access_log')}} {{ '${' + slave_directory_section + ':log-folder}' }}/apache-access.log
# Set password for slave
{% set slave_password_section = slave_reference + "-password" -%}
[{{slave_password_section}}]
recipe = slapos.cookbook:generate.password
storage-path = {{apache_configuration_directory}}/.{{slave_reference}}.passwd
bytes = 8
# Set up htaccess file for slave
{% set slave_htaccess_section = slave_reference + '-htaccess' %}
{% do part_list.append(slave_htaccess_section) -%}
[{{slave_htaccess_section}}]
recipe = plone.recipe.command
stop-on-error = true
htaccess-path = {{apache_configuration_directory}}/.{{slave_reference}}.htaccess
command = {{frontend_configuration.get('apache-directory')}}/bin/htpasswd -cb ${:htaccess-path} {{ slave_reference }} {{ '${' + slave_password_section + ':passwd}' }}
# Add slave log directory to the slave log access dict
{% do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
# Set up apache configuration file for slave
[{{ slave_section_title }}]
< = jinja2-template-base
......@@ -72,18 +131,37 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
{% endif -%}
# Publish slave information
{% set slave_log_access_url = 'https://' + slave_reference + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('apache-ipv6') + ']:' + frontend_configuration.get('apache-https-port') + '/' + slave_reference.lower() + '/' %}
{% if not extra_slave_instance_list -%}
{% set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
{% do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
public-ipv4 = {{ public_ipv4 }}
log-access = {{ slave_log_access_url }}
-slave-reference = {{ slave_instance.get('slave_reference') }}
{% else -%}
{% do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4}) -%}
{% do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'log-access': slave_log_access_url}) -%}
{% endif -%}
{% endfor -%}
[slave-log-directories]
{% for key, value in slave_log_dict.iteritems() -%}
{{ key }} = {{ value }}
{% endfor %}
# Define log access
{% set log_access_section = "apache-log-access" %}
{% do part_list.append(log_access_section) -%}
[{{log_access_section}}]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directories
raw apache_log_directory {{apache_log_directory}}
raw apache_configuration_directory {{apache_configuration_directory}}
# Publish information for the instance
{% set publish_section_title = 'publish-apache-information' -%}
{% do part_list.append(publish_section_title) -%}
......
......@@ -64,6 +64,11 @@ LoadModule headers_module {{ httpd_home }}/modules/mod_headers.so
LoadModule cache_module {{ httpd_home }}/modules/mod_cache.so
LoadModule mem_cache_module {{ httpd_home }}/modules/mod_mem_cache.so
LoadModule antiloris_module {{ httpd_home }}/modules/mod_antiloris.so
LoadModule alias_module {{ httpd_home }}/modules/mod_alias.so
LoadModule autoindex_module {{ httpd_home }}/modules/mod_autoindex.so
LoadModule auth_basic_module {{ httpd_home }}/modules/mod_auth_basic.so
LoadModule authz_user_module {{ httpd_home }}/modules/mod_authz_user.so
LoadModule authn_file_module {{ httpd_home }}/modules/mod_authn_file.so
# The following directives modify normal HTTP response behavior to
# handle known problems with browser implementations.
......@@ -125,6 +130,8 @@ SSLCipherSuite RC4-SHA:HIGH:!ADH
SSLProxyCheckPeerCN off
SSLProxyCheckPeerExpire off
include {{frontend_configuration.get('log-access-configuration')}}
NameVirtualHost *:{{ http_port }}
NameVirtualHost *:{{ https_port }}
include {{ slave_configuration_directory }}/*.conf
\ No newline at end of file
......@@ -6,8 +6,15 @@
{% set slave_list = json_module.loads(slave_list_raw) -%}
{% for slave_dict in slave_list -%}
{% set slave_reference = slave_dict.pop('slave-reference') %}
{% set log_access_url = slave_dict.pop('log-access', '') %}
{% set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
{% do current_slave_dict.update(slave_dict) -%}
{% set log_access_list = current_slave_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend + ': ' + log_access_url) %}
{% do current_slave_dict.__setitem__(
'log-access-urls',
log_access_list
) %}
{% do current_slave_dict.__setitem__(
'replication_number',
current_slave_dict.get('replication_number', 0) + 1
......@@ -23,6 +30,7 @@
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
log-access-url = {{ json_module.dumps(slave_information.pop('log-access-urls', 1000)) }}
{% for key, value in slave_information.iteritems() -%}
{{ key }} = {{ value }}
{% endfor -%}
......
{% for slave, directory in slave_log_directory.iteritems() %}
Alias /{{slave}}/ {{directory}}/
<Directory {{directory}}>
Order Deny,Allow
Deny from env=AUTHREQUIRED
<Files ".??*">
Order Allow,Deny
Deny from all
</Files>
AuthType Basic
AuthName "Log Access {{slave}}"
AuthUserFile "{{ apache_configuration_directory + '/.' + slave.upper() + '.htaccess'}}"
Require user {{slave.upper()}}
Options Indexes FollowSymLinks
Satisfy all
</Directory>
{% endfor %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment