apache-frontend: Publish url per frontend per slave to access logs

apache-frontend: log access, one per slave

apache-frontend: protect log access with password

apache-frontend: slave publish their log-access-url

apache-frontend: publish all log-access urls

apache-frontend: comment md5 for development

apache-frontend: slapos.recipe.cmmi updated

apache-frontend: fix log access

apache-frontend: specify frontend for the log access

apache-frontend: update md5sum

apache-frontend: update versions

software/apache-frontend/common.cfg

@@ -36,6 +36,7 @@ recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   slapos.toolbox
+  plone.recipe.command
 
 scripts =
   killpidfromfile
@@ -58,7 +59,7 @@ mode = 0644
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = 9f3eec91f43ae0730e9bba93f83572fc
+md5sum = f0a507fed2b1dcab5530c892adce8327
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
@@ -71,7 +72,7 @@ mode = 0644
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = f5eef006211809669b12422240c6f436
+md5sum = f002a8fc8fc5d18adbd8ac1ee054e852
 mode = 640
 
 [template-slave-configuration]
@@ -83,13 +84,13 @@ mode = 640
 [template-replicate-publish-slave-information]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/replicate-publish-slave-information.cfg.in
-md5sum = 61a14dff06718e3d90c346a0a7b20c5a
+md5sum = a2cf00b24877bf747e2408d444f16f05
 mode = 640
 
 [template-apache-frontend-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache.conf.in
-md5sum = b4d7461c29fb6c36f09e48fa5ad59fba
+md5sum = bae89ebc6c5e75b12535fbd6c37f647d
 mode = 640
 
 [template-apache-cached-configuration]
@@ -129,6 +130,12 @@ url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
 md5sum = ac845c0fa3835832307a0e7323cb339d
 mode = 640
 
+[template-log-access]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/template-log-access.conf.in
+md5sum = f85005b430978f3bd24ee7ce11b0e304
+mode = 640
+
 [template-squid-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/squid.conf.jinja2

software/apache-frontend/instance-apache-frontend.cfg

@@ -27,6 +27,7 @@ parts =
   promise-apache-frontend-v6-http
   promise-apache-cached
 
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -87,6 +88,13 @@ configuration.apache-certificate =
 configuration.open-port = 80 443
 configuration.extra_slave_instance_list =
 
+[frontend-configuration]
+template-log-access = ${template-log-access:target}
+log-access-configuration = $${directory:etc}/apache-log-access.conf
+apache-directory = ${apache-2.2:location}
+apache-ipv6 = $${instance-parameter:ipv6-random}
+apache-https-port = $${instance-parameter:configuration.port}
+
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = $${buildout:directory}/$${:filename}
@@ -129,6 +137,9 @@ extra-context =
     raw template_slave_configuration ${template-slave-configuration:target}
     raw template_rewrite_cached ${template-rewrite-cached:target}
     raw software_type single-custom-personal
+    section logrotate_dict logrotate
+    section frontend_configuration frontend-configuration
+    section apache_configuration apache-configuration
 
 [dynamic-custom-group-template-slave-list]
 < = jinja2-template-base
@@ -214,6 +225,7 @@ extra-context =
     key error_log apache-configuration:error-log
     key pid_file apache-configuration:pid-file
     key slave_configuration_directory apache-directory:slave-configuration
+    section frontend_configuration frontend-configuration
 
 [apache-frontend]
 recipe = slapos.cookbook:wrapper
@@ -366,7 +378,7 @@ state-file = $${directory:srv}/logrotate.status
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
 name = apache
-log = $${apache-directory:slave-log}/*_log $${apache-configuration:error-log} $${apache-configuration:access-log}
+log = $${apache-configuration:error-log} $${apache-configuration:access-log}
 frequency = daily
 rotatep-num = 30
 post = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:pid-file} SIGUSR1
@@ -425,7 +437,7 @@ pid-filename-path = $${directory:run}/squid.pid
 template = ${template-squid-configuration:target}
 rendered = $${squid-cache:configuration-path}
 extra-context =
-      key ip squid-cache:ip 
+      key ip squid-cache:ip
       key port squid-cache:port
       key backend_ip squid-cache:backend-ip
       key backend_port squid-cache:backend-port

software/apache-frontend/software.cfg

@@ -13,7 +13,7 @@ meld3 = 0.6.10
 pycrypto = 2.6
 rdiff-backup = 1.0.5
 slapos.recipe.build = 0.11.6
-slapos.recipe.cmmi = 0.1
+slapos.recipe.cmmi = 0.1.1
 slapos.recipe.template = 2.4.2
 slapos.toolbox = 0.34.0
 smmap = 0.8.2
@@ -23,7 +23,15 @@ cmd2 = 0.6.5.1
 prettytable = 0.7.2
 requests = 1.2.3
 slapos.cookbook = 0.82
+cffi = 0.8.2
+cryptography = 0.3
+plone.recipe.command = 1.1
+pyOpenSSL = 0.14
+six = 1.6.1
 
+# Required by:
+# cffi==0.8.2
+pycparser = 2.10
 # Required by:
 # slapos.cookbook==0.82
 lock-file = 2.0

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -4,8 +4,9 @@
 {% set part_list = [] -%}
 {% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
 {% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
+{% set slave_log_dict = {} -%}
 {% if extra_slave_instance_list -%}
-{%   set slave_instance_information_list = []-%}
+{%   set slave_instance_information_list = [] -%}
 {%   set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
 {% endif -%}
 [jinja2-template-base]
@@ -17,15 +18,73 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     ${:extra-context}
 
+{% do logrotate_dict.pop('recipe') %}
+[logrotate]
+{% for key, value in logrotate_dict.iteritems() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+
 # Loop trhought slave list to set up slaves
 {% for slave_instance in slave_instance_list -%}
 {%   set slave_reference = slave_instance.get('slave_reference') -%}
 {%   set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference -%}
 {%   set slave_parameter_dict = generic_instance_parameter_dict.copy() -%}
 {%   do part_list.append(slave_section_title) -%}
+
+
+{%   set slave_directory_section = slave_reference + "-directory" -%}
+{%   set slave_log_folder = logrotate_dict.get('backup') + '/' + slave_reference + "-logs" -%}
+[{{slave_directory_section}}]
+recipe = slapos.cookbook:mkdirectory
+log-folder = {{slave_log_folder}}
+
 # Set Up log files
 {%   do slave_parameter_dict.__setitem__('access_log', '/'.join([apache_log_directory, '%s_access_log' % slave_reference])) -%}
 {%   do slave_parameter_dict.__setitem__('error_log', '/'.join([apache_log_directory, '%s_error_log' % slave_reference])) -%}
+
+# Set slave logrotate entry
+{%   set slave_logrotate_section = slave_reference + "-logs" -%}
+{%   do part_list.append(slave_logrotate_section) -%}
+[{{slave_logrotate_section}}]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = ${:_buildout_section_name_}
+log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
+backup = {{ '${' + slave_directory_section + ':log-folder}' }}
+frequency = daily
+rotatep-num = 30
+post = ${buildout:bin-directory}/killpidfromfile {{ apache_configuration.get('pid-file') }} SIGUSR1
+sharedscripts = true
+notifempty = true
+create = true
+
+# integrate current logs inside
+{%   set slave_ln_section = slave_reference + "-ln" -%}
+{%   do part_list.append(slave_ln_section) -%}
+[{{slave_ln_section}}]
+recipe = plone.recipe.command
+stop-on-error = false
+command = ln -s {{slave_parameter_dict.get('error_log')}} {{ '${' + slave_directory_section + ':log-folder}' }}/apache-error.log && ln -s {{slave_parameter_dict.get('access_log')}} {{ '${' + slave_directory_section + ':log-folder}' }}/apache-access.log
+
+# Set password for slave
+{%   set slave_password_section = slave_reference + "-password" -%}
+[{{slave_password_section}}]
+recipe = slapos.cookbook:generate.password
+storage-path = {{apache_configuration_directory}}/.{{slave_reference}}.passwd
+bytes = 8
+
+# Set up htaccess file for slave
+{%   set slave_htaccess_section = slave_reference + '-htaccess' %}
+{%   do part_list.append(slave_htaccess_section) -%}
+[{{slave_htaccess_section}}]
+recipe = plone.recipe.command
+stop-on-error = true
+htaccess-path = {{apache_configuration_directory}}/.{{slave_reference}}.htaccess
+command = {{frontend_configuration.get('apache-directory')}}/bin/htpasswd -cb ${:htaccess-path} {{ slave_reference }} {{ '${' + slave_password_section + ':passwd}' }}
+
+# Add slave log directory to the slave log access dict
+{%   do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
+
 # Set up apache configuration file for slave
 [{{ slave_section_title }}]
 < = jinja2-template-base
@@ -72,18 +131,37 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
 {%   endif -%}
 
 # Publish slave information
+{%   set slave_log_access_url = 'https://' + slave_reference + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('apache-ipv6') + ']:' + frontend_configuration.get('apache-https-port') + '/' + slave_reference.lower() + '/' %}
 {%   if not extra_slave_instance_list -%}
 {%     set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
 {%     do part_list.append(publish_section_title) -%}
 [{{ publish_section_title }}]
 recipe = slapos.cookbook:publish
 public-ipv4 = {{ public_ipv4 }}
+log-access = {{ slave_log_access_url }}
 -slave-reference = {{ slave_instance.get('slave_reference') }}
 {%   else -%}
-{%     do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4}) -%}
+{%     do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'log-access': slave_log_access_url}) -%}
 {%   endif -%}
 {% endfor -%}
 
+[slave-log-directories]
+{% for key, value in slave_log_dict.iteritems() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+
+# Define log access
+{% set log_access_section = "apache-log-access" %}
+{% do part_list.append(log_access_section) -%}
+[{{log_access_section}}]
+< = jinja2-template-base
+template = {{frontend_configuration.get('template-log-access')}}
+rendered = {{frontend_configuration.get('log-access-configuration')}}
+extra-context =
+    section slave_log_directory slave-log-directories
+    raw apache_log_directory {{apache_log_directory}}
+    raw apache_configuration_directory {{apache_configuration_directory}}
+
 # Publish information for the instance
 {% set publish_section_title = 'publish-apache-information' -%}
 {% do part_list.append(publish_section_title) -%}

software/apache-frontend/templates/apache.conf.in

@@ -64,6 +64,11 @@ LoadModule headers_module     {{ httpd_home }}/modules/mod_headers.so
 LoadModule cache_module       {{ httpd_home }}/modules/mod_cache.so
 LoadModule mem_cache_module   {{ httpd_home }}/modules/mod_mem_cache.so
 LoadModule antiloris_module   {{ httpd_home }}/modules/mod_antiloris.so
+LoadModule alias_module       {{ httpd_home }}/modules/mod_alias.so
+LoadModule autoindex_module   {{ httpd_home }}/modules/mod_autoindex.so
+LoadModule auth_basic_module  {{ httpd_home }}/modules/mod_auth_basic.so
+LoadModule authz_user_module  {{ httpd_home }}/modules/mod_authz_user.so
+LoadModule authn_file_module  {{ httpd_home }}/modules/mod_authn_file.so
 
 # The following directives modify normal HTTP response behavior to
 # handle known problems with browser implementations.
@@ -125,6 +130,8 @@ SSLCipherSuite RC4-SHA:HIGH:!ADH
 SSLProxyCheckPeerCN off
 SSLProxyCheckPeerExpire off
 
+include {{frontend_configuration.get('log-access-configuration')}}
+
 NameVirtualHost *:{{ http_port }}
 NameVirtualHost *:{{ https_port }}
 include {{ slave_configuration_directory }}/*.conf
\ No newline at end of file

software/apache-frontend/templates/replicate-publish-slave-information.cfg.in

@@ -6,8 +6,15 @@
 {%   set slave_list = json_module.loads(slave_list_raw) -%}
 {%   for slave_dict in slave_list -%}
 {%     set slave_reference = slave_dict.pop('slave-reference') %}
+{%     set log_access_url = slave_dict.pop('log-access', '') %}
 {%     set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
 {%     do current_slave_dict.update(slave_dict) -%}
+{%     set log_access_list = current_slave_dict.get('log-access-urls', []) %}
+{%     do log_access_list.append( frontend + ': ' + log_access_url) %}
+{%     do current_slave_dict.__setitem__(
+            'log-access-urls',
+            log_access_list
+            ) %}
 {%     do current_slave_dict.__setitem__(
             'replication_number',
             current_slave_dict.get('replication_number', 0) + 1
@@ -23,6 +30,7 @@
 [{{ publish_section_title }}]
 recipe = slapos.cookbook:publish
 -slave-reference = {{ slave_reference }}
+log-access-url = {{ json_module.dumps(slave_information.pop('log-access-urls', 1000)) }}
 {%   for key, value in slave_information.iteritems() -%}
 {{ key }} = {{ value }}
 {%   endfor -%}

software/apache-frontend/templates/template-log-access.conf.in

+{% for slave, directory in slave_log_directory.iteritems() %}
+Alias /{{slave}}/ {{directory}}/
+<Directory {{directory}}>
+  Order Deny,Allow
+  Deny from env=AUTHREQUIRED
+  <Files ".??*">
+    Order Allow,Deny
+    Deny from all
+  </Files>
+  AuthType Basic
+  AuthName "Log Access {{slave}}"
+  AuthUserFile "{{ apache_configuration_directory + '/.' + slave.upper() + '.htaccess'}}"
+  Require user {{slave.upper()}}
+  Options Indexes FollowSymLinks
+  Satisfy all
+</Directory>
+
+{% endfor %}