stack/erp5: support frontend-caucase-url-list option.

software/erp5/instance-erp5-input-schema.json

@@ -455,6 +455,16 @@
         "ssl": {
           "description": "HTTPS certificate generation parameters",
           "properties": {
+            "frontend-caucase-url-list": {
+              "title": "Frontend Caucase URL List",
+              "description": "List of URLs of caucase service of frontend groups to authenticate access from them.",
+              "type": "array",
+              "items": {
+                "type": "string",
+                "format": "uri"
+              },
+              "uniqueItems": true
+            },
             "caucase-url": {
               "title": "Caucase URL",
               "description": "URL of caucase service to use. If not set, global setting will be used.",

software/erp5/test/test/test_balancer.py

+from . import ERP5InstanceTestCase
+from . import setUpModule
+from slapos.testing.utils import findFreeTCPPort
+
+from BaseHTTPServer import HTTPServer
+from BaseHTTPServer import BaseHTTPRequestHandler
+import OpenSSL.SSL
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization, hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography import x509
+from cryptography.x509.oid import NameOID
+import hashlib
+import json
+import multiprocessing
+import os
+import requests
+import shutil
+import subprocess
+import tempfile
+import time
+
+setUpModule  # pyflakes
+
+class TestHandler(BaseHTTPRequestHandler):
+  def do_GET(self):
+    self.send_response(200)
+    self.send_header("Content-Type", "application/json")
+    response = {
+      'Path': self.path,
+      'Incoming Headers': self.headers.dict
+    }
+    response = json.dumps(response, indent=2)
+    self.end_headers()
+    self.wfile.write(response)
+
+class TestFrontendXForwardedFor(ERP5InstanceTestCase):
+  http_server_process = None
+  frontend_caucase_dir = None
+  frontend_caucased_process = None
+  backend_caucase_dir = None
+  backend_caucased_process = None
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'balancer'
+
+  @classmethod
+  def setUpClass(cls):
+    # start a dummy web server echoing headers.
+    http_server_port = findFreeTCPPort(cls._ipv4_address)
+    server = HTTPServer(
+      (cls._ipv4_address, http_server_port),
+      TestHandler)
+    cls.http_server_process = multiprocessing.Process(
+      target=server.serve_forever, name='HTTPServer')
+    cls.http_server_process.start()
+    cls.http_server_netloc = '%s:%s' % (cls._ipv4_address, http_server_port)
+
+    # start a caucased and generate a valid client certificate.
+    cls.computer_partition_root_path = os.path.abspath(os.curdir)
+    cls.frontend_caucase_dir = tempfile.mkdtemp()
+    frontend_caucased_dir = os.path.join(cls.frontend_caucase_dir, 'caucased')
+    os.mkdir(frontend_caucased_dir)
+    frontend_user_dir = os.path.join(cls.frontend_caucase_dir, 'user')
+    os.mkdir(frontend_user_dir)
+    frontend_service_dir = os.path.join(cls.frontend_caucase_dir, 'service')
+    os.mkdir(frontend_service_dir)
+    frontend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
+    cls.frontend_caucased_url = 'http://' + frontend_caucased_netloc
+
+    cls.user_certificate = frontend_user_key = os.path.join(frontend_user_dir, 'client.key.pem')
+    frontend_user_csr = os.path.join(frontend_user_dir, 'client.csr.pem')
+
+    key = rsa.generate_private_key(
+      public_exponent=65537,
+      key_size=2048,
+      backend=default_backend()
+    )
+    with open(frontend_user_key, 'wb') as f:
+      f.write(key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+      ))
+
+    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
+      x509.NameAttribute(NameOID.COMMON_NAME, u'user'),
+    ])).sign(key, hashes.SHA256(), default_backend())
+    with open(frontend_user_csr, 'wb') as f:
+      f.write(csr.public_bytes(serialization.Encoding.PEM))
+
+    cls.software_release_root_path = os.path.join(
+       cls.slap._software_root,
+       hashlib.md5(cls.getSoftwareURL()).hexdigest(),
+    )
+    caucased_path = os.path.join(cls.software_release_root_path, 'bin', 'caucased')
+    caucase_path = os.path.join(cls.software_release_root_path, 'bin', 'caucase')
+    cls.frontend_caucased_process = subprocess.Popen(
+      [
+        caucased_path,
+        '--db', os.path.join(frontend_caucased_dir, 'caucase.sqlite'),
+        '--server-key', os.path.join(frontend_caucased_dir, 'server.key.pem'),
+        '--netloc', frontend_caucased_netloc,
+        '--service-auto-approve-count', '1',
+      ],
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+    )
+    for _ in range(10):
+      try:
+        if requests.get(cls.frontend_caucased_url).status_code == 200:
+          break
+      except Exception:
+        pass
+      time.sleep(1)
+    else:
+      raise RuntimeError, 'caucased failed to start.'
+
+    cau_args = [
+      caucase_path,
+      '--ca-url', cls.frontend_caucased_url,
+      '--ca-crt', os.path.join(frontend_user_dir, 'service-ca-crt.pem'),
+      '--crl', os.path.join(frontend_user_dir, 'service.crl'),
+      '--user-ca-crt', os.path.join(frontend_user_dir, 'user-ca-crt.pem'),
+      '--user-crl', os.path.join(frontend_user_dir, 'user.crl'),
+    ]
+
+    cas_args = [
+      caucase_path,
+      '--ca-url', cls.frontend_caucased_url,
+      '--ca-crt', os.path.join(frontend_service_dir, 'service-ca-crt.pem'),
+      '--crl', os.path.join(frontend_service_dir, 'service.crl'),
+      '--user-ca-crt', os.path.join(frontend_service_dir, 'user-ca-crt.pem'),
+      '--user-crl', os.path.join(frontend_service_dir, 'user.crl'),
+    ]
+
+    caucase_process = subprocess.Popen(
+      cau_args + [
+        '--mode', 'user',
+        '--send-csr', frontend_user_csr,
+      ],
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+    )
+    result = caucase_process.communicate()
+    print result
+    csr_id = result[0].split()[0]
+
+    subprocess.check_call(
+      cau_args + [
+        '--mode', 'user',
+        '--get-crt', csr_id, frontend_user_key,
+      ],
+    )
+
+    cls.client_certificate = frontend_service_key = os.path.join(frontend_service_dir, 'crt.pem')
+    frontend_service_csr = os.path.join(frontend_service_dir, 'csr.pem')
+
+    key = rsa.generate_private_key(
+      public_exponent=65537,
+      key_size=2048,
+      backend=default_backend()
+    )
+    with open(frontend_service_key, 'wb') as f:
+      f.write(key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+      ))
+
+    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
+      x509.NameAttribute(NameOID.COMMON_NAME, u'service'),
+    ])).sign(key, hashes.SHA256(), default_backend())
+    with open(frontend_service_csr, 'wb') as f:
+      f.write(csr.public_bytes(serialization.Encoding.PEM))
+
+    caucase_process = subprocess.Popen(
+      cas_args + [
+        '--send-csr', frontend_service_csr,
+      ],
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+    )
+    result = caucase_process.communicate()
+    csr_id = result[0].split()[0]
+
+    for _ in range(10):
+      if not subprocess.call(
+        cas_args + [
+          '--get-crt', csr_id, frontend_service_key,
+        ],
+      ) == 0:
+        break
+      else:
+        time.sleep(1)
+    else:
+      raise RuntimeError, 'getting service certificate failed.'
+
+    # start a caucased and server certificate.
+    cls.backend_caucase_dir = tempfile.mkdtemp()
+    backend_caucased_dir = os.path.join(cls.backend_caucase_dir, 'caucased')
+    os.mkdir(backend_caucased_dir)
+    backend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
+    cls.backend_caucased_url = 'http://' + backend_caucased_netloc
+    cls.backend_caucased_process = subprocess.Popen(
+      [
+        caucased_path,
+        '--db', os.path.join(backend_caucased_dir, 'caucase.sqlite'),
+        '--server-key', os.path.join(backend_caucased_dir, 'server.key.pem'),
+        '--netloc', backend_caucased_netloc,
+        '--service-auto-approve-count', '1',
+      ],
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+    )
+    for _ in range(10):
+      try:
+        if requests.get(cls.backend_caucased_url).status_code == 200:
+          break
+      except Exception:
+        pass
+      time.sleep(1)
+    else:
+      raise RuntimeError, 'caucased failed to start.'
+
+    super(TestFrontendXForwardedFor, cls).setUpClass()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      '_': json.dumps({
+        'tcpv4-port': 3306,
+        'computer-memory-percent-threshold': 100,
+        # XXX what is this ? should probably not be needed here
+        'name': cls.__name__,
+        'monitor-passwd': 'secret',
+        'apachedex-configuration': '',
+        'apachedex-promise-threshold': 100,
+        'haproxy-server-check-path': '/',
+        'zope-family-dict': {
+          'default': ['dummy_http_server'],
+          'default-auth': ['dummy_http_server'],
+        },
+        'dummy_http_server': [[cls.http_server_netloc, 1, False]],
+        'backend-path-dict': {
+          'default': '/',
+          'default-auth': '/',
+        },
+        'ssl-authentication-dict': {
+          'default': False,
+          'default-auth': True,
+        },
+        'ssl': {
+          'caucase-url': cls.backend_caucased_url,
+          'frontend-caucase-url-list': [cls.frontend_caucased_url],
+        },
+      })
+    }
+
+  @classmethod
+  def _cleanup(cls, snapshot_name):
+    if cls.http_server_process:
+      cls.http_server_process.terminate()
+    if cls.frontend_caucased_process:
+      cls.frontend_caucased_process.terminate()
+    if cls.frontend_caucase_dir:
+      shutil.rmtree(cls.frontend_caucase_dir)
+    if cls.backend_caucased_process:
+      cls.backend_caucased_process.terminate()
+    if cls.backend_caucase_dir:
+      shutil.rmtree(cls.backend_caucase_dir)
+
+    super(TestFrontendXForwardedFor, cls)._cleanup(snapshot_name)
+
+  def test_x_forwarded_for_added_when_verified_connection(self):
+    for backend in ('default', 'default-auth'):
+      balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])[backend]
+      result = requests.get(
+        balancer_url,
+        headers={'X-Forwarded-For': '1.2.3.4'},
+        cert=self.client_certificate,
+        verify=False,
+      ).json()
+      self.assertEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
+
+  def test_x_forwarded_for_stripped_when_not_verified_connection(self):
+    balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
+    result = requests.get(
+      balancer_url,
+      headers={'X-Forwarded-For': '1.2.3.4'},
+      verify=False,
+    ).json()
+    self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
+    balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
+    with self.assertRaises(OpenSSL.SSL.Error):
+      requests.get(
+        balancer_url,
+        headers={'X-Forwarded-For': '1.2.3.4'},
+        verify=False,
+      )

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = f0f3b18f9963b137e366752886591fc3
+md5sum = 328ea2bb5f2bff18f8be8c541c01f260
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -90,7 +90,7 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 28b68d6eb1af5a48b25b05a21919db2a
+md5sum = 0097e49b5bd7ad4978c722c1cdd27d6c
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -2,6 +2,7 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
+{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
 per partition. No more (undefined result), no less (IndexError).
@@ -38,6 +39,64 @@ mode = 644
 {% do section('caucase-updater') -%}
 {% do section('caucase-updater-promise') -%}
 
+{% set frontend_caucase_url_hash_list = [] -%}
+{% for frontend_caucase_url in frontend_caucase_url_list -%}
+{%   set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%}
+{%   do frontend_caucase_url_hash_list.append(hash) -%}
+{%   set data_dir = '${directory:srv}/client-cert-ca/%s' % hash -%}
+{{   caucase.updater(
+       prefix='caucase-updater-%s' % hash,
+       buildout_bin_directory=parameter_dict['bin-directory'],
+       updater_path='${directory:services-on-watch}/caucase-updater-%s' % hash,
+       url=frontend_caucase_url,
+       data_dir=data_dir,
+       ca_path='%s/ca.crt' % data_dir,
+       crl_path='%s/crl.pem' % data_dir,
+       on_renew='${caucase-updater-housekeeper:output}; ${apache-graceful:output}',
+       max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
+       openssl=parameter_dict['openssl'] ~ '/bin/openssl',
+     )}}
+{%   do section('caucase-updater-%s' % hash) -%}
+{% endfor -%}
+
+{% if frontend_caucase_url_hash_list -%}
+[caucase-updater-housekeeper]
+recipe = collective.recipe.template
+output = ${directory:bin}/caucase-updater-housekeeper
+mode = 700
+input =
+  inline:
+  #!${buildout:executable}
+  import glob
+  import os
+  import subprocess
+  hash_list = {{ repr(frontend_caucase_url_hash_list) }}
+  crt_list = ['%s.crt' % e for e in hash_list]
+  crl_list = ['%s.crl' % e for e in hash_list]
+  for path in glob.glob('${apache-conf-ssl:ca-cert-dir}/*.crt'):
+    if os.path.basename(path) not in crt_list:
+      os.unlink(path)
+  for path in glob.glob('${apache-conf-ssl:crl-dir}/*.crl'):
+    if os.path.basename(path) not in crl_list:
+      os.unlink(path)
+  for hash in hash_list:
+    crt = '${directory:srv}/client-cert-ca/%s/ca.crt' % hash
+    crt_link = '${apache-conf-ssl:ca-cert-dir}/%s.crt' % hash
+    crl = '${directory:srv}/client-cert-ca/%s/crl.pem' % hash
+    crl_link = '${apache-conf-ssl:crl-dir}/%s.crl' % hash
+    if os.path.isfile(crt) and not os.path.islink(crt_link):
+      os.symlink(crt, crt_link)
+    if os.path.isfile(crl) and not os.path.islink(crl_link):
+      os.symlink(crl, crl_link)
+  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:ca-cert-dir}'])
+  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:crl-dir}'])
+
+[caucase-updater-housekeeper-run]
+recipe = plone.recipe.command
+command = ${caucase-updater-housekeeper:output}
+update-command = ${:command}
+{% endif -%}
+
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
@@ -123,6 +182,27 @@ key = ${directory:apache-conf}/apache.pem
 # XXX caucase certificate is not supported by caddy for now
 caucase-cert = ${directory:apache-conf}/apache-caucase.crt
 caucase-key = ${directory:apache-conf}/apache-caucase.pem
+{% if frontend_caucase_url_list -%}
+depends = ${caucase-updater-housekeeper-run:recipe}
+ca-cert-dir = ${directory:apache-ca-cert-dir}
+crl-dir = ${directory:apache-crl-dir}
+{%- endif %}
+
+[simplefile]
+< = jinja2-template-base
+template = inline:{{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+rendered = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
 
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
@@ -152,9 +232,11 @@ cert = ${apache-ssl:cert}
 key = ${apache-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
+{% if frontend_caucase_url_list -%}
 # Client x509 auth
-ca-cert = ${apache-conf-ssl:ca-cert}
-crl = ${apache-conf-ssl:crl}
+ca-cert-dir = ${apache-conf-ssl:ca-cert-dir}
+crl-dir = ${apache-conf-ssl:crl-dir}
+{%- endif %}
 
 [apache-conf]
 < = jinja2-template-base
@@ -207,6 +289,10 @@ post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bi
 [directory]
 recipe = slapos.cookbook:mkdirectory
 apache-conf = ${:etc}/apache
+{% if frontend_caucase_url_list -%}
+apache-ca-cert-dir = ${:apache-conf}/ssl.crt
+apache-crl-dir = ${:apache-conf}/ssl.crl
+{% endif -%}
 bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
 services = ${:etc}/run

stack/erp5/instance.cfg.in

@@ -72,6 +72,7 @@ filename = instance-balancer.cfg
 extra-context =
     section parameter_dict dynamic-template-balancer-parameters
     import itertools itertools
+    import hashlib hashlib
 import-list =
     file caucase context:caucase-jinja2-library