Add new stack certificate authority based on new CA implemetation

The CA python egg is here: https://lab.nexedi.com/vpelletier/caucase
instance-certificate-authority.cfg.jinja2.in deploy a CA server which expose an API on HTTP,
all request are done using GET, PUT, DELETE and POST on that API.

Auth server is an apache httpd which validate client certificate for authentification.
It autmatically request a signed certificate to CA and use it in apache configuration.

client request will be validated using:
SSLVerifyClient require in apache config

stack/certificate-authority/buildout.cfg

+[buildout]
+
+extends =
+  ../../component/apache/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/pycurl/buildout.cfg
+  ../../component/python-cryptography/buildout.cfg
+  ../../stack/logrotate/buildout.cfg
+
+
+parts =
+  template-logrotate-base
+
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = python.ca
+eggs =
+  gunicorn # for WSGI HTTP Server
+  futures
+  certificate.authority
+  plone.recipe.command
+  collective.recipe.template
+
+[template-ca-download-base]
+recipe = hexagonit.recipe.download
+ignore-existing = true
+download-only = true
+url = ${:_profile_base_location_}/template/${:filename}
+mode = 0644
+
+[template-httpd-auth-conf]
+<= template-ca-download-base
+md5sum = ea445b0a9b143d12b5700a71ac06293c
+filename = template-httpd-auth.conf.in
+
+[template-nginx-ca-conf]
+<= template-ca-download-base
+md5sum = 97d8dbcdfcca92a3c2b70634f0dee8d9
+filename = ca-nginx.conf.in
+
+[template-authenticated-server]
+recipe = slapos.recipe.template:jinja2
+filename = template-authenticated-server.cfg
+template = ${:_profile_base_location_}/instance-auth-server.cfg.jinja2.in
+rendered = ${buildout:directory}/template-authenticated-server.cfg
+md5sum = 39c1494b45dcbd5388b0d1c1d9b27ffb
+context =
+  key apache_location apache:location
+  key gzip_location gzip:location
+  key template_logrotate_base template-logrotate-base:rendered
+  raw certificate_request_bin ${buildout:directory}/bin/ca-cliweb
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw dcron_executable_location ${dcron:location}/sbin/crond
+  raw slapos_kill_bin ${buildout:directory}/bin/slapos-kill
+  raw template_httpd_auth_conf ${template-httpd-auth-conf:location}/${template-httpd-auth-conf:filename}
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  raw python_executable ${buildout:directory}/bin/${extra-eggs:interpreter}
+
+[template-certificate-authority]
+recipe = slapos.recipe.template:jinja2
+filename = template-certificate-authority.cfg
+template = ${:_profile_base_location_}/instance-certificate-authority.cfg.jinja2.in
+rendered = ${buildout:directory}/template-certificate-authority.cfg
+md5sum = d5139f650388256776f43b9026617564
+context =
+  key ngix_location nginx:location
+  key template_logrotate_base template-logrotate-base:rendered
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw certificate_authority_bin ${buildout:directory}/bin/ca-bin
+  raw template_nginx_ca_conf ${template-nginx-ca-conf:location}/${template-nginx-ca-conf:filename}
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw gunicorn_bin ${buildout:directory}/bin/gunicorn
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  raw python_bin ${buildout:directory}/bin/${extra-eggs:interpreter}
+  raw eggs_directory  ${buildout:eggs-directory}
+  raw develop_eggs_directory ${buildout:develop-eggs-directory}

stack/certificate-authority/instance-auth-server.cfg.jinja2.in

+[buildout]
+
+extends =
+  {{ template_logrotate_base }}
+
+parts = 
+  authenticated-httpd-server
+
+[authenticated-server-parameters]
+ca-url = 
+common-name = instance@${slap-configuration:instance-title}
+server-port = 8286
+custom-httpd-file = 
+web-directory = ${directory:document-root}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+ssl = ${:etc}/ssl
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+document-root = ${:srv}/private
+
+[certificate-request-base]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/request-instance-certificate
+cert-file = ${directory:ssl}/instance.cert.pem
+key-file = ${directory:ssl}/instance.key.pem
+ca-cert = ${directory:ssl}/cacert.pem
+parameters-extra = true
+command-line = {{ certificate_request_bin }}
+  --crt-file ${:cert-file}
+  --key-file ${:key-file}
+  --ca-url ${authenticated-server-parameters:ca-url}
+  --ca-crt-file ${:ca-cert}
+  --no-check-certificate
+
+[server-certificate-request]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/request-instance-certificate
+command-line =
+  ${certificate-request-base:wrapper-path}
+  --cn ${authenticated-server-parameters:common-name}
+  --request
+
+[authenticated-httpd-conf-parameter]
+ip = ${slap-configuration:ipv6-random}
+port = ${authenticated-server-parameters:server-port}
+pid-file = ${directory:run}/httpd-auth.pid
+dav-lock = ${directory:var}/DavLockdb
+access-log = ${directory:log}/httpd-auth-access.log
+error-log = ${directory:log}/httpd-auth-error.log
+cert-file = ${certificate-request-base:cert-file}
+key-file = ${certificate-request-base:key-file}
+ca-cert = ${certificate-request-base:ca-cert}
+url = https://[${:ip}]:${:port}
+private = ${authenticated-server-parameters:web-directory}
+httpd-include-file = ${authenticated-server-parameters:custom-httpd-file}
+crl = 
+
+[authenticated-httpd-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_httpd_auth_conf }}
+rendered = ${directory:etc}/httpd-auth.conf
+mode = 0744
+context =
+  section parameter_dict authenticated-httpd-conf-parameter
+
+[authenticated-httpd-graceful]
+recipe = collective.recipe.template
+input = inline:
+  #!{{ dash_executable_location }}
+  kill -USR1 $(cat ${authenticated-httpd-conf-parameter:pid-file})
+
+output = ${directory:scripts}/authenticated-httpd-graceful
+mode = 700
+
+[authenticated-httpd-server]
+recipe = slapos.cookbook:wrapper
+command-line = {{ apache_location }}/bin/httpd -f ${authenticated-httpd-conf:rendered} -DFOREGROUND
+wrapper-path = ${directory:services}/authenticated-httpd-server
+wait-for-files =
+  ${certificate-request-base:cert-file}
+  ${certificate-request-base:key-file}
+  ${certificate-request-base:ca-cert}
+
+url = ${authenticated-httpd-conf-parameter:url}
+
+depends = 
+  ${authenticated-httpd-promise:filename}
+  ${authenticated-httpd-graceful:output}
+  ${server-certificate-request:wrapper-path}
+  ${logrotate-authenticated-httpd:name}
+  ${certificate-renew-cron-entry:name}
+
+[certificate-renew]
+recipe = collective.recipe.template
+input = inline:
+  #!{{ dash_executable_location }}
+
+  d=$({{ openssl_executable_location }} x509 -enddate -noout -in ${certificate-request-base:cert-file} | cut -d'=' -f 2)
+  cert_time=$(date -d "$d" +"%s")
+  now=$(date +"%s")
+  thresold=2592000  # 30*24*60*60  equivalent to one month in seconds
+  remind=$(($cert_time - $now))
+
+  if [ $remind -lt $thresold ]; then
+    exec ${certificate-request-base:wrapper-path} --renew
+  fi
+
+output = ${directory:bin}/certificate-renew
+mode = 700
+
+[certificate-renew-cron-entry]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = certificate-auto-renew
+frequency = 0 */2 * * *
+command = ${certificate-renew:output}
+
+[logrotate-authenticated-httpd]
+< = logrotate-entry-base
+name = authenticated-httpd-server
+log = ${authenticated-httpd-conf-parameter:access-log} ${authenticated-httpd-conf-parameter:access-log}
+post = {{ slapos_kill_bin }} --pidfile ${authenticated-httpd-conf-parameter:pid-file} -s USR1
+
+[authenticated-httpd-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/${:filename}
+filename = authenticated-httpd-is-available
+url = ${authenticated-httpd-conf-parameter:url}
+check-secure = 1
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+cert-file = ${certificate-request-base:cert-file}
+key-file = ${certificate-request-base:key-file}
+ca-cert-file = ${certificate-request-base:ca-cert}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}

stack/certificate-authority/instance-certificate-authority.cfg.jinja2.in

+[buildout]
+
+extends =
+  {{ template_logrotate_base }}
+
+parts = 
+  certificate-authority
+  certificate-authority-server
+
+[certificate-authority-parameters]
+server-port = 8009
+# Overrite this to set frontend URL
+external-url = https://[${slap-configuration:ipv6-random}]:${:server-port}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+ssl=${:etc}/ssl
+ca-dir = ${directory:srv}/ca
+ca-temp = ${:ca-dir}/tmp
+client-body-temp-path = ${:ca-temp}/client_body_temp_path
+proxy-temp-path = ${:ca-temp}/proxy_temp_path
+fastcgi-temp-path = ${:ca-temp}/fastcgi_temp_path
+uwsgi-temp-path = ${:ca-temp}/uwsgi_temp_path
+scgi-temp-path = ${:ca-temp}/scgi_temp_path
+
+[ca-directory]
+recipe = slapos.cookbook:mkdirectory
+root = ${directory:srv}/ssl
+requests = ${:root}/requests
+private = ${:root}/private
+certs = ${:root}/certs
+newcerts = ${:root}/newcerts
+crl = ${:root}/crl
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_executable_location }}
+ca-dir = ${ca-directory:root}
+requests-directory = ${ca-directory:requests}
+wrapper = ${directory:services}/certificate_authority
+ca-private = ${ca-directory:private}
+ca-certs = ${ca-directory:certs}
+ca-newcerts = ${ca-directory:newcerts}
+ca-crl = ${ca-directory:crl}
+
+[ca-nginx-ssl]
+recipe = plone.recipe.command
+# XXX - For now, generate ca httpd certificate here, because it's not possible to start CA without this files
+command = "{{ openssl_executable_location }}" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
+key = ${directory:ssl}/ca-cert.key
+cert = ${directory:ssl}/ca-cert.crt
+update-command = 
+stop-on-error = true
+
+[ca-nginx-conf-parameter]
+ip = ${slap-configuration:ipv6-random}
+port = ${certificate-authority-parameters:server-port}
+pid-file = ${directory:run}/nginx-ca.pid
+access-log = ${directory:log}/nginx-ca-access.log
+error-log = ${directory:log}/nginx-ca-error.log
+cert-file = ${ca-nginx-ssl:cert}
+key-file = ${ca-nginx-ssl:key}
+ca-conf = ${certificate-authority-conf:output}
+workers-processes = 1
+client-body-temp-path = ${directory:client-body-temp-path}
+proxy-temp-path = ${directory:proxy-temp-path}
+fastcgi-temp-path = ${directory:fastcgi-temp-path}
+uwsgi-temp-path = ${directory:uwsgi-temp-path}
+scgi-temp-path = ${directory:scgi-temp-path}
+socket = ${certificate-authority-gunicorn:socket}
+
+[ca-nginx-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_nginx_ca_conf }}
+rendered = ${directory:etc}/nginx-ca.conf
+mode = 0700
+context =
+  section parameter_dict ca-nginx-conf-parameter
+
+[certificate-authority-conf]
+recipe = collective.recipe.template
+input = inline:
+  ca-dir ${directory:ca-dir}
+  # enable debug
+  # debug
+  # log-file ${directory:log}/ca-server.log
+  subject /C=XX/ST=State/L=City/OU=OUnit/O=Company/CN=SlapOS Certificate Authority/emailAddress=xx@example.com
+  max-request-amount 10
+  external-url ${certificate-authority-parameters:external-url}
+  # one year (in seconds)
+  crt-life-time 31536000
+  # crl-life-period correspond to about one week
+  crl-life-period 0.02
+  # ca-life-time = ca-life-period * crt-life-time
+  ca-life-period 10
+
+output = ${directory:etc}/ca.conf
+mode = 700
+
+[ca-nginx-graceful]
+recipe = collective.recipe.template
+input = inline:#!{{ dash_executable_location }}
+  kill -HUP $(cat ${ca-nginx-conf-parameter:pid-file})
+
+output = ${directory:scripts}/ca-server-graceful
+mode = 700
+
+[certificate-authority-gunicorn]
+recipe = slapos.cookbook:wrapper
+socket = ${directory:ca-dir}/ca.flaskserver.sock
+command-line =  {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${certificate-authority-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload
+log-file = ${directory:log}/ca-gunicorn-error.log
+pid-file = ${directory:run}/ca-gunicorn.pid
+wrapper-path = ${directory:services}/ca-gunicorn
+#environment = #PATH=$${environ:PATH}:${git:location}/bin/
+#  CA_CONFIGURATION_FILE=${certificate-authority-conf:output}
+#  LANG=en_GB.UTF-8
+
+[certificate-authority-server]
+recipe = slapos.cookbook:wrapper
+command-line = {{ ngix_location }}/sbin/nginx -p ${directory:ca-dir} -c ${ca-nginx-conf:rendered}
+wrapper-path = ${directory:services}/ca-server
+wait-for-files =
+  ${ca-nginx-ssl:cert}
+  ${ca-nginx-ssl:key}
+#environment = 
+#  CA_CONFIGURATION_FILE=${certificate-authority-conf:output}
+url = ${certificate-authority-parameters:external-url}
+depends = 
+  ${certificate-authority-server-promise:filename}
+  ${ca-nginx-graceful:output}
+
+[certificate-authority-server-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/${:filename}
+filename = certificate-authority-server-listening-on-tcp
+url = http://[${slap-configuration:ipv6-random}]:${certificate-authority-parameters:server-port}
+check-secure = 1
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-parameter]

stack/certificate-authority/template/ca-nginx.conf.in

+worker_processes {{ parameter_dict['workers-processes'] }};
+
+pid {{ parameter_dict['pid-file'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+    # include mime.types;
+    default_type application/octet-stream;
+    access_log {{ parameter_dict['access-log'] }} combined;
+    client_max_body_size 10M;
+    map $http_upgrade $connection_upgrade {
+      default upgrade;
+      ''      close;
+    }
+    sendfile on;
+
+    upstream app_server {
+      # for UNIX domain socket setups
+      server unix:{{ parameter_dict['socket'] }} fail_timeout=0;
+    }
+
+     server {
+        listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }} ssl;
+        server_name _;
+        ssl_certificate     {{ parameter_dict['cert-file'] }};
+        ssl_certificate_key {{ parameter_dict['key-file'] }};
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers on;
+        keepalive_timeout 90s;
+        client_body_temp_path {{ parameter_dict['client-body-temp-path'] }};
+        proxy_temp_path {{ parameter_dict['proxy-temp-path'] }};
+        fastcgi_temp_path {{ parameter_dict['fastcgi-temp-path'] }};
+        uwsgi_temp_path {{ parameter_dict['uwsgi-temp-path'] }};
+        scgi_temp_path {{ parameter_dict['scgi-temp-path'] }};
+
+        location / {
+            proxy_redirect off;
+            proxy_set_header   X-Forwarded-Proto $scheme;
+            proxy_set_header   X-Real-IP $remote_addr;
+            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Host  $http_host;
+            proxy_set_header   Host $http_host;
+            proxy_set_header   Authorization $http_authorization;
+            proxy_pass_header  Authorization;
+            proxy_connect_timeout 90;
+            proxy_send_timeout    90;
+            proxy_read_timeout    90;
+            send_timeout          90;
+
+            proxy_pass http://app_server;
+        }
+    }
+}

stack/certificate-authority/template/template-httpd-auth.conf.in

+Listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }}
+
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule dir_module modules/mod_dir.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule autoindex_module modules/mod_autoindex.so
+LoadModule proxy_module      modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule env_module modules/mod_env.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
+
+ServerAdmin admin@
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+ServerTokens Prod
+ServerSignature Off
+TraceEnable Off
+
+PidFile "{{ parameter_dict['pid-file'] }}"
+ErrorLog "{{ parameter_dict['error-log'] }}"
+# Default apache log format with request time in microsecond at the end
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+CustomLog "{{ parameter_dict['access-log'] }}" combined
+
+# SSL Configuration
+Define SSLConfigured
+SSLCertificateFile {{ parameter_dict['cert-file'] }}
+SSLCertificateKeyFile {{ parameter_dict['key-file'] }}
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+SSLRandomSeed startup /dev/urandom 256
+SSLRandomSeed connect builtin
+SSLProtocol all -SSLv2 -SSLv3
+SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
+SSLHonorCipherOrder on
+
+SSLEngine   On
+
+
+<Directory />
+  Options FollowSymLinks
+  AllowOverride None
+  Allow from all
+</Directory>
+
+<VirtualHost *:{{ parameter_dict['port'] }}>
+
+  DavLockDB {{ parameter_dict['dav-lock'] }}
+
+  SSLVerifyClient require
+  RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+  SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+  {% if parameter_dict['crl'] -%}
+  SSLCARevocationCheck chain
+  SSLCARevocationFile {{ parameter_dict['crl'] }}
+  {%- endif %}
+
+  Alias / {{ parameter_dict['private'] }}/
+  <Directory {{ parameter_dict['private'] }}>
+    DirectoryIndex disabled
+    DAV On
+    Options Indexes FollowSymLinks
+    Order Allow,Deny
+    Allow from all
+  </Directory>
+
+</VirtualHost>
+
+{% if parameter_dict.get('httpd-include-file', '') -%}
+# Custom apache configuration here
+Include {{ parameter_dict['httpd-include-file'] }}
+{% endif -%}

[Vincent Pelletier]

Why both apache and nginx ?

[Vincent Pelletier]

Is it really required ? I would expect python-cryptography to pull it implicitly.

[Alain Takoudjou]

probably i used it to generate certificate ssl for nginx (which should be changed). I will check and remove if not needed

[Vincent Pelletier]

Although this SR is still simple, maybe it could use the buildout.hash.cfg approach ? Just a suggestion in case you missed it. Details are at: https://lab.nexedi.com/nexedi/slapos/blob/master/update-hash (docstring) and an example would be: nexedi/slapos@5f0f8d53 (so you can see what changes it requires/allows).

[Alain Takoudjou]

yes, thanks

[Vincent Pelletier]

Common templating bits may benefit from being factorised.

[Vincent Pelletier]

Overall looks good (modulo questions for what I noticed and could not understand).I did not review nginx & apache configuratoins seriously (because I am bad at configuring apache and I never configured nginx).

Once (usual) issue I have with slapos: there is so much duplication betwen SRs... Especially, these cipher suites duplicated everywhere are annoying: the day this value becomes considered unsafe, we have many places to fix.

[Vincent Pelletier]

Ah, and something else, which is also a common issue I have with slapos: I do not know how easy it will be for other SRs (ex: ERP5) to reuse these. You may want to check what julien did for neo SR, as it is extended by ERP5.

[Vincent Pelletier]

...and I should also add that I like that you use overall very few distinct recipes, and all of them are also used by other SRs. This is how I expect recipes to be used, thanks !