...
 
Commits (113)
Showing 200 changed files with 1140 additions and 624 deletions
[buildout]
parts = babeld
[babeld-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/babeld.git
branch = master
git-executable = ${git:location}/bin/git
revision = hmac-nxd1
[babeld]
recipe = slapos.recipe.cmmi
url = https://lab.nexedi.com/nexedi/babeld/repository/archive.tar.gz?ref=v1.8.4-nxd1
md5sum = 536cca6abe34eb9faeda8682b80ee7bc
path = ${babeld-repository:location}
make-options = CC='gcc -std=gnu99'
configure-command =
echo "No configure.."
......
......@@ -12,9 +12,9 @@ parts =
[ca-certificates]
recipe = slapos.recipe.cmmi
shared = true
url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20170717.tar.xz
url = http://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20190110.tar.xz
md5sum = e91d3d9259127ba2dbb65fda58d73f31
patch-binary = ${patch:location}/bin/patch
md5sum = 55a6bb6b98afb16b3cde8e3ad1e262eb
patches =
${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
......
[buildout]
extends =
../../component/golang/buildout.cfg
gowork.cfg
parts =
gowork
caddy
[gowork]
# Caddy 1.x+ uses go modules, for which gowork does not work yet
golang = ${golang1.12:location}
install =
github.com/mholt/caddy
[gowork.goinstall]
command = :
depends =
${caddy:recipe}
[caddy]
recipe = slapos.recipe.cmmi
path = ${go_github.com_mholt_caddy:location}
go = ${gowork:golang}/bin/go
configure-command = :
make-targets =
make-binary = cd ${:path}/caddy && ${:go} install -v
environment =
PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory}
# revision and repository can be used to control which caddy version is used
revision = v1.0.0
repository = github.com/mholt/caddy/caddy
recipe = plone.recipe.command
update-command = ${:command}
stop-on-error = True
# GO111MODULE=on enables go modules support
# the chmod is needed as modules are fetched with u-w
command =
. ${gowork:env.sh} &&
cd ${gowork:directory} &&
export GO111MODULE=on &&
go get ${:repository}@${:revision} &&
chmod -R u+w .
output = ${gowork:bin}/caddy
location = ${:output}
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_mholt_caddy:recipe}
[go_github.com_mholt_caddy]
<= go-git-package
go.importpath = github.com/mholt/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v0.11.5-4-g9d3151db
......@@ -17,5 +17,5 @@ rpath = ${:library-dirs}
[geolite2-country]
recipe = slapos.recipe.build:download-unpacked
url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
md5sum = 81c75ee8c3baab3c99591c9a99f94530
md5sum = ff7bcb609291b9f5ab14397ccc506254
strip-top-level-dir = true
......@@ -64,8 +64,8 @@ environment-extra =
[golang1.12]
<= golang-common
url = https://dl.google.com/go/go1.12.4.src.tar.gz
md5sum = edb6bac6dc1ce0afc931d9e41ef3d2ef
url = https://dl.google.com/go/go1.12.6.src.tar.gz
md5sum = 48a4141fc718dd742d106431294f08bf
# go1.11 needs go1.4 to bootstrap
environment-extra =
......
......@@ -29,11 +29,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.2.23
md5sum = 941c9ac6ee7709fd88a4098ecfc0a4b0
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
version = 10.3.14
md5sum = b1b9628bbc3ff15e8f5cfc7896c73975
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = ${cmake:location}/bin/cmake
configure-options =
......@@ -75,7 +72,6 @@ environment =
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
......
# 33_scripts__mysql_create_system_tables__no_test.dpatch by <ch@debian.org>
A user with no password prevents a normal user from login under certain
circumstances as it is checked first.
See http://bugs.debian.org/301741
and http://bugs.mysql.com/bug.php?id=6901
--- scripts/mysql_system_tables_data.sql.orig 2018-06-25 11:13:26.000000000 +0000
+++ scripts/mysql_system_tables_data.sql 2018-06-29 10:50:42.032343572 +0000
@@ -30,8 +30,6 @@
-- Fill "db" table with default grants for anyone to
-- access database 'test' and 'test_%' if "db" table didn't exist
CREATE TEMPORARY TABLE tmp_db LIKE db;
-INSERT INTO tmp_db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
-INSERT INTO tmp_db VALUES ('%','test\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0;
DROP TABLE tmp_db;
@@ -51,9 +49,6 @@
IF @auth_root_socket is not null THEN
IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN
INSTALL SONAME 'auth_socket'; END IF; END IF;
--- Anonymous user with no privileges.
-INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost','');
-INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost';
INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL;
INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL;
......@@ -98,7 +98,7 @@ configure-options=
--with-http_v2_module
--with-http_gzip_static_module
--with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
--with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include"
--with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include -Wno-error"
--add-module=${nginx-push-stream-module:location}
[nginx-push-stream-output]
......
......@@ -7,5 +7,5 @@ parts =
[perl-URI]
<= perl-CPAN-package
module = URI/URI
version = 1.72
md5sum = cd56d81ed429efaa97e7f3ff08851b48
version = 1.76
md5sum = c236e0142adecc1b1104da664bc43a79
......@@ -7,8 +7,11 @@ url = https://download.pureftpd.org/pub/pure-ftpd/releases/pure-ftpd-1.0.46.tar.
md5sum = efce5529c1f0a39dafdd532c619503f1
# See https://download.pureftpd.org/pub/pure-ftpd/doc/README for more configurations
# We need the trick about UPLOAD_PIPE_FILE and UPLOAD_PIPE_LOCK so that the files are created inside the $CWD/var/run
# WARNING: this means that both pure-ftpd and pure-uploadscript binaries must be launched in $HOME !
configure-options =
--with-uploadscript
#environment =
# Probably it is missing dependencies to be set here.
--with-puredb
--with-nonroot
environment=
CFLAGS=-DUPLOAD_PIPE_FILE='"/proc/self/cwd/var/run/pure-ftpd.upload.pipe"' -DUPLOAD_PIPE_LOCK='"/proc/self/cwd/var/run/pure-ftpd.upload.lock"'
[buildout]
extends = ../xz-utils/buildout.cfg
parts = backports.lzma
[backports.lzma]
recipe = zc.recipe.egg:custom
egg = backports.lzma
rpath = ${xz-utils:location}/lib
library-dirs = ${:rpath}
include-dirs = ${xz-utils:location}/include
......@@ -17,16 +17,20 @@ import collections
def main():
exit_code = 0
for f in sys.argv[1:]:
print 'Processing %s' % (f,)
with open(f, 'rb') as infile:
try:
obj = json.load(infile, object_pairs_hook=collections.OrderedDict)
except ValueError as e:
raise SystemExit(e)
with open(f, 'wb') as outfile:
json.dump(obj, outfile, sort_keys=False, indent=2, separators=(',', ': '))
outfile.write('\n')
exit_code = 1
print e
else:
with open(f, 'wb') as outfile:
json.dump(obj, outfile, sort_keys=False, indent=2, separators=(',', ': '))
outfile.write('\n')
sys.exit(exit_code)
if __name__ == '__main__':
main()
......@@ -43,7 +43,7 @@ dir = %(working_directory)s # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
unique_subject = no # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
......
......@@ -26,6 +26,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import errno
import io
import logging
import os
......@@ -33,6 +34,7 @@ import sys
import inspect
import re
import shutil
import stat
import urllib
import urlparse
......@@ -92,9 +94,27 @@ class GenericBaseRecipe(object):
"""Create a file with content
The parent directory should exists, else it would raise IOError"""
with open(name, 'w') as fileobject:
fileobject.write(content)
os.chmod(fileobject.name, mode)
if not isinstance(content, bytes):
content = content.encode('utf-8')
# Try to reuse existing file. This is particularly
# important to avoid excessive IO during update.
try:
with open(name, 'rb') as f:
if f.read(len(content)+1) == content:
if None is not mode != stat.S_IMODE(os.fstat(f.fileno()).st_mode):
os.fchmod(f.fileno(), mode)
return os.path.abspath(name)
except (IOError, OSError) as e:
pass
try:
os.unlink(name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
with open(name, 'wb') as f:
if mode is not None:
os.fchmod(f.fileno(), mode)
f.write(content)
return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700):
......
......@@ -24,18 +24,17 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import json
import re
import logging, os
import zc.buildout.easy_install
from pprint import pformat
from slapos.recipe.librecipe import GenericBaseRecipe
script_template = '''# This script is auto generated by slapgrid, do not edit!
import json
import sys
sys.path[0:0] = %(path)s
extra_config_dict = json.loads("""%(config)s""", strict=False)
extra_config_dict = %(config)s
# We want to cleanup all imported modules from slapos namespace, because
# they will conflict with slapos.core.
......@@ -69,12 +68,9 @@ class Recipe(GenericBaseRecipe):
"""Return a mapping where to store generated working sets.
from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
"""
cache_storage = getattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
None
)
if cache_storage is None:
try:
return getattr(self.buildout, self._WORKING_SET_CACHE_NAME)
except AttributeError:
cache_storage = {}
try:
setattr(
......@@ -83,29 +79,29 @@ class Recipe(GenericBaseRecipe):
cache_storage
)
except AttributeError:
if type(self.buildout) == type({}):
# failed to set attribute in test mode, cache not used
pass
else:
if not isinstance(self.buildout, dict):
raise
# failed to set attribute in test mode, cache not used
return cache_storage
def install(self):
develop_eggs_dir = self.options['develop-eggs-directory']
eggs_dir = self.options['eggs-directory']
egg_list = [
egg_list = tuple(
egg.strip()
for egg in self.options['eggs'].split('\n')
for egg in self.options['eggs'].splitlines()
if egg.strip()
]
)
cache_storage = self._get_cache_storage()
cache_key = (
tuple(egg_list),
egg_list,
eggs_dir,
develop_eggs_dir,
)
if cache_key not in cache_storage:
try:
working_set = cache_storage[cache_key]
except KeyError:
if develop_eggs_dir and eggs_dir:
working_set = zc.buildout.easy_install.working_set(
egg_list,
......@@ -114,8 +110,6 @@ class Recipe(GenericBaseRecipe):
cache_storage[cache_key] = working_set
else:
working_set = set()
else:
working_set = cache_storage[cache_key]
regex = r"^[\w_\-\.\s]+$"
import_path = self.options.get('import', '').strip()
......@@ -129,24 +123,14 @@ class Recipe(GenericBaseRecipe):
if not re.search(regex, content_string):
raise ValueError("Promise content %r is not valid" % content_string)
output = self.options['output']
mode = self.options.get('mode', '0644')
path_list = []
for dist in working_set:
path_list.append(dist.location)
config_dict = dict()
for key in self.options:
if key.startswith('config-'):
config_dict[key[7:]] = self.options[key]
option_dict = dict(path=json.dumps(path_list, indent=2),
content=content_string,
config=json.dumps(config_dict, indent=2, sort_keys=True))
with open(output, 'w') as f:
f.write(script_template % option_dict)
config_dict = {key[7:]: self.options[key]
for key in self.options
if key.startswith('config-')}
os.chmod(output, int(mode, 8))
return (output,)
return self.createFile(self.options['output'], script_template % {
'path': pformat([dist.location for dist in working_set], indent=2),
'content': content_string,
'config': pformat(config_dict, indent=2),
}, int(self.options.get('mode', '0644'), 8)),
update = install
......@@ -93,7 +93,7 @@ def requestRemoveToken(client, token_base_path):
reference = reference_key.split('.')[0]
try:
result = client.deleteToken(token)
except httplib.NOTFOUND:
except httplib.NOT_FOUND:
# Token is alread removed.
result = True
except Exception:
......
......@@ -92,6 +92,27 @@ SSL keys and certificates are directly send to the frontend cluster in order to
*Note*: Until master partition or slave specific certificate is uploaded each slave is served with fallback certificate. This fallback certificate is self signed, does not match served hostname and results with lack of response on HTTPs.
Obtaining CA for KeDiFa
-----------------------
KeDiFa uses caucase and so it is required to obtain caucase CA certificate used to sign KeDiFa SSL certificate, in order to be sure that certificates are sent to valid KeDiFa.
The easiest way to do so is to use caucase.
On some secure and trusted box which will be used to upload certificate to master or slave frontend partition install caucase https://pypi.org/project/caucase/
Master and slave partition will return key ``kedifa-caucase-url``, so then create and start a ``caucase-updater`` service::
caucase-updater \
--ca-url "${kedifa-caucase-url}" \
--cas-ca "${frontend_name}.caucased.ca.crt" \
--ca "${frontend_name}.ca.crt" \
--crl "${frontend_name}.crl"
where ``frontend_name`` is a frontend cluster to which you will upload the certificate (it can be just one slave).
Make sure it is automatically started when trusted machine reboots: you want to have it running so you can forget about it. It will keep KeDiFa's CA certificate up to date when it gets renewed so you know you are still talking to the same service as when you previously uploaded the certificate, up to the original upload.
Master partition
----------------
......@@ -105,12 +126,12 @@ Example sessions is::
request(...)
curl -k -X GET master-key-generate-auth-url
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" master-key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
curl -k -X PUT --data-binary @master.pem master-key-upload-url+authtoken
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem master-key-upload-url+authtoken
This replaces old request parameters:
......@@ -133,12 +154,12 @@ Example sessions is::
request(...)
curl -k -X GET key-generate-auth-url
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
curl -k -X PUT --data-binary @master.pem key-upload-url+authtoken
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem key-upload-url+authtoken
This replaces old request parameters:
......
Generally things to be done with ``caddy-frontend``:
* README: cleanup the documentation, explain various specifics
* check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
* (new) ``type:websocket`` slave
* ``type:eventsource``:
* **Jérome Perrin**: *For event source, if I understand https://github.com/mholt/caddy/issues/1355 correctly, we could use caddy as a proxy in front of nginx-push-stream . If we have a "central shared" caddy instance, can it handle keeping connections opens for many clients ?*
* ``check-error-on-caddy-log`` like ``check-error-on-apache-log``
* move out ``test/utils.py`` and use it from shared python distribution
* reduce the time of configuration validation (in ``instance-apache-frontend.cfg.in`` sections ``[configtest]``, ``[caddy-configuration]``), as it is not scalable on frontend with 2000+ slaves (takes few minutes instead of few, < 5, seconds), issue posted `upstream <https://github.com/mholt/caddy/issues/2220>`_
* drop ``6tunnel`` and use ``bind`` in Caddy configuration, as soon as multiple binds will be possible, tracked in upstream `bind: support multiple values <https://github.com/mholt/caddy/pull/2128>`_ and `ipv6: does not bind on ipv4 and ipv6 for sites that resolve to both <https://github.com/mholt/caddy/issues/864>`_
* use caddy-frontend in `standalone style playbooks <https://lab.nexedi.com/nexedi/slapos.package/tree/master/playbook/roles/standalone-shared>`_
* in ``templates/apache-custom-slave-list.cfg.in`` avoid repetetive ``part_list.append`` and use macro like in ERP5 SR (cf `Vincent's comment <https://lab.nexedi.com/nexedi/slapos/merge_requests/373#note_64362>`_)
* **Jérome Perrin**: consider privacy implications/GDPR compliance of https://caddyserver.com/docs/telemetry and decide if we should leave it enabled.
......@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 6abe3aae72a83ef90e484d00cc70490d
md5sum = 72e8ff0773fd0325dcbe994786156570
[template-apache-replicate]
[template-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = d62aefe002ec13875924e4c219914795
md5sum = 6ef06ef36dbb0c3ba1ea8841bcd49330
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 75439cb035393e68c73672b224bead54
md5sum = 3faad78c80b0e9235849f0b5346eebbc
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -38,7 +38,7 @@ md5sum = 54ae95597a126ae552c3a913ddf29e5e
[template-replicate-publish-slave-information]
filename = templates/replicate-publish-slave-information.cfg.in
md5sum = 38e9994be01ea1b8a379f8ff7aa05438
md5sum = eb9ca67763d60843483d95dab2c301b1
[template-caddy-frontend-configuration]
filename = templates/Caddyfile.in
......@@ -58,7 +58,7 @@ md5sum = 0c5ef7f26a142c3ab53e835d2caa698d
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
md5sum = 907372828d1ceb05c41240078196f439
md5sum = 6ca9a3251830d602cf25e0a0389fc74b
[template-log-access]
filename = templates/template-log-access.conf.in
......@@ -74,7 +74,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[template-trafficserver-records-config]
filename = templates/trafficserver/records.config.jinja2
md5sum = 15cbdba7ab7fe3c3d7c26c1718e47bbd
md5sum = 5ef0ebc37437ada7cc176e663da5f36c
[template-trafficserver-storage-config]
filename = templates/trafficserver/storage.config.jinja2
......@@ -82,7 +82,7 @@ md5sum = baf7b89cc9ab5506100b0c900808c1ea
[template-trafficserver-logging-config]
filename = templates/trafficserver/logging.config.jinja2
md5sum = cd6bb9bd0734f17469b0ca88f8b1a531
md5sum = 0f30ca0d299d0150b22fe8b0ee12f150
[template-nginx-eventsource-slave-virtualhost]
filename = templates/nginx-eventsource-slave.conf.in
......@@ -94,11 +94,19 @@ md5sum = b9f73f6323f9fceea054c46c854d2862
[template-graceful-script]
filename = templates/graceful-script.sh.in
md5sum = 31dd34de4c40a1d814bcbef482c757cc
md5sum = 061cc244558fd3af2b6bacf17cae5555
[template-validate-script]
filename = templates/validate-script.sh.in
md5sum = 4c80ba3727c397f8d5b12cf1888cf959
md5sum = f26e11574f266c7437c9c89e3c93825a
[template-configuration-state-script]
filename = templates/configuration-state-script.sh.in
md5sum = 4d2537d2698d32a7e909989f8778d144
[template-rotate-script]
filename = templates/rotate-script.sh.in
md5sum = 8c150e1e6c993708d31936742f3a7302
[caddyprofiledeps-setup]
filename = setup.py
......@@ -110,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
filename = instance-kedifa.cfg.in
md5sum = cc6f32656e76f4b79b5e47567b930f74
md5sum = 299cea88789c8904e0e4414a55fd3a5c
......@@ -9,6 +9,7 @@ extends =
../../component/rdiff-backup/buildout.cfg
../../component/trafficserver/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../stack/nodejs.cfg
../../stack/caucase/buildout.cfg
......@@ -35,7 +36,7 @@ parts +=
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = 73a14b0e88afe7512f2fefe6ee9e0000fa523d5d
revision = c89c31a5cdc9367e196411d91c4dcd0229bb7893
[kedifa-develop]
recipe = zc.recipe.egg:develop
......@@ -113,12 +114,15 @@ sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
xz_location = ${xz-utils:location}
monitor_template = ${monitor-template:output}
template_cached_slave_virtualhost = ${template-cached-slave-virtualhost:target}
template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
template_graceful_script = ${template-graceful-script:target}
template_validate_script = ${template-validate-script:target}
template_rotate_script = ${template-rotate-script:target}
template_configuration_state_script = ${template-configuration-state-script:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
......@@ -237,6 +241,10 @@ filename = storage.config.jinja2
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
filename = logging.config.jinja2
[template-rotate-script]
<=download-template
filename = rotate-script.sh.in
[template-caddy-lazy-script-call]
<=download-template
filename = apache-lazy-script-call.sh.in
......@@ -249,6 +257,10 @@ filename = graceful-script.sh.in
<=download-template
filename = validate-script.sh.in
[template-configuration-state-script]
<=download-template
filename = configuration-state-script.sh.in
# Migrated from KVM recipe
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
......
......@@ -9,7 +9,6 @@ extends =
parts =
directory
configtest
logrotate-entry-caddy
caddy-frontend
switch-caddy-softwaretype
......@@ -36,7 +35,7 @@ parts =
trafficserver-logging-config
trafficserver-promise-listen-port
trafficserver-promise-cache-availability
cron-entry-logrotate-trafficserver
## Monitor for Caddy
monitor-base
monitor-ats-cache-stats-wrapper
......@@ -245,6 +244,7 @@ extra-context =
key autocert caddy-directory:autocert
key master_certificate caddy-configuration:master-certificate
key caddy_log_directory caddy-directory:slave-log
key caddy_log_cache_direct_directory caddy-directory:slave-log-cache-direct
key local_ipv4 :local_ipv4
key local_ipv6 :local_ipv6
key global_ipv6 slap-network-information:global-ipv6
......@@ -344,7 +344,9 @@ wrapper-path = ${directory:bin}/caddy-wrapper
recipe = slapos.cookbook:wrapper
command-line = ${caddy-wrapper:wrapper-path} -pidfile ${caddy-configuration:pid-file}
wrapper-path = ${directory:service}/frontend_caddy
hash-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
${buildout:directory}/software_release/buildout.cfg
${caddy-wrapper:wrapper-path}
[not-found-html]
recipe = slapos.cookbook:symbolic.link
......@@ -360,6 +362,7 @@ slave-with-cache-configuration = ${directory:etc}/caddy-slave-with-cache-conf.d/
cache = ${directory:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
slave-log = ${directory:log}/httpd
slave-log-cache-direct = ${directory:log}/httpd-cache-direct
autocert = ${directory:srv}/autocert
master-autocert-dir = ${:autocert}/master-autocert
custom-ssl-directory = ${:slave-configuration}/ssl
......@@ -369,8 +372,7 @@ frontend-configuration = ${directory:etc}/Caddyfile
access-log = ${directory:log}/frontend-access.log
error-log = ${directory:log}/frontend-error.log
pid-file = ${directory:run}/httpd.pid
frontend-configuration-verification = ${frontend-caddy-validate:rendered}
frontend-graceful-command = ${:frontend-configuration-verification} && kill -USR1 $(cat ${:pid-file})
frontend-graceful-command = ${frontend-caddy-validate:rendered} && kill -USR1 $(cat ${:pid-file})
not-found-file = ${caddy-directory:document-root}/notfound.html
master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
# Communication with ATS
......@@ -378,11 +380,6 @@ cache-port = ${trafficserver-variable:input-port}
cache-through-port = 26011
ssl-cache-through-port = 26012
[configtest]
recipe = slapos.cookbook:wrapper
command-line = ${frontend-caddy-validate:rendered}
wrapper-path = ${directory:bin}/caddy-configtest
# BBB: SlapOS Master non-zero knowledge BEGIN
[get-self-signed-fallback-access]
recipe = collective.recipe.shelloutput
......@@ -425,6 +422,7 @@ local-state = ${directory:var}/trafficserver
bin_path = {{ parameter_dict['trafficserver'] }}/bin
log = ${directory:log}/trafficserver
cache-path = ${directory:srv}/ats_cache
logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
[trafficserver-variable]
wrapper-path = ${directory:service}/trafficserver
......@@ -539,36 +537,75 @@ module = trafficserver_cache_availability
name = trafficserver-cache-availability.py
config-wrapper-path = ${trafficserver-ctl:wrapper-path}
[trafficserver-rotate-script]
< = jinja2-template-base
template = {{ parameter_dict['template_rotate_script'] }}
rendered = ${directory:bin}/trafficserver-rotate
mode = 0700
xz_binary = {{ parameter_dict['xz_location'] ~ '/bin/xz' }}
pattern = *.old
# days to keep log files
keep_days = 365
extra-context =
key log_dir trafficserver-directory:log
key rotate_dir trafficserver-directory:logrotate-backup
key xz_binary :xz_binary
key keep_days :keep_days
key pattern :pattern
[cron-entry-logrotate-trafficserver]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = trafficserver-logrotate
frequency = 0 0 * * *
command = ${trafficserver-rotate-script:rendered}
### End of ATS sections
### Caddy Graceful and promises
[frontend-caddy-graceful]
[frontend-caddy-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
template = {{ parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/caddy_graceful_signature
extra-context =
key graceful_reload_command caddy-configuration:frontend-graceful-command
key path_list :path_list
key sha256sum :sha256sum
key signature_file :signature_file
[frontend-caddy-configuration-state-graceful]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/graceful_configuration_state_signature
[frontend-caddy-configuration-state-validate]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/validate_configuration_state_signature
[frontend-caddy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
mode = 0700
extra-context =
key graceful_reload_command caddy-configuration:frontend-graceful-command
key caddy_configuration_state frontend-caddy-configuration-state-graceful:rendered
[frontend-caddy-validate]
< = jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/frontend-caddy-validate
mode = 0700
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/caddy_validate_signature
last_state_file = ${directory:run}/caddy_configuration_last_state
extra-context =
key wrapper caddy-wrapper:wrapper-path
key path_list frontend-caddy-graceful:path_list
key sha256sum :sha256sum
key signature_file :signature_file
key caddy_configuration_state frontend-caddy-configuration-state-validate:rendered
key last_state_file :last_state_file
[frontend-caddy-lazy-graceful]
< = jinja2-template-base
......@@ -583,11 +620,22 @@ extra-context =
key lazy_command caddy-configuration:frontend-graceful-command
# Promises checking configuration:
[promise-helper-last-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
rendered = ${directory:bin}/frontend-read-last-configuration-state
mode = 0700
content =
#!/bin/sh
exit `cat ${frontend-caddy-validate:last_state_file}`
context =
key content :content
[promise-frontend-caddy-configuration]
<= promise-plugin-base
module = validate_frontend_configuration
name = frontend-caddy-configuration-promise.py
config-verification-script = ${caddy-configuration:frontend-configuration-verification}
config-verification-script = ${promise-helper-last-configuration-state:rendered}
[promise-caddy-frontend-v4-https]
<= promise-plugin-base
......
......@@ -72,6 +72,7 @@ context =
{% set authorized_slave_string_list = slapparameter_dict.pop('-frontend-authorized-slave-string', '').split() %}
{% set authorized_slave_list = [] %}
{% set rejected_slave_dict = {} %}
{% set rejected_slave_title_dict = {} %}
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% set unauthorized_message = 'slave not authorized' %}
......@@ -133,6 +134,14 @@ context =
{% do slave_error_list.append('slave https-url %r invalid' % (slave['https-url'],)) %}
{% endif %}
{% endif %}
{% set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt') %}
{% if ssl_proxy_ca_crt %}
{% set check_popen = popen([parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(ssl_proxy_ca_crt) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
{% endif %}
{% endif %}
{# BBB: SlapOS Master non-zero knowledge BEGIN #}
{% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{% if key in slave %}
......@@ -167,6 +176,7 @@ context =
{% do authorized_slave_list.append(slave) %}
{% else %}
{% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), slave_error_list) %}
{% do rejected_slave_title_dict.__setitem__(slave.get('slave_title'), slave_error_list) %}
{% endif %}
{% if len(slave_warning_list) > 0 %}
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), slave_warning_list) %}
......@@ -231,7 +241,8 @@ domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_dict | length }}
rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict)) }}
rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_title_dict)) }}
rejected-slave-promise-url = ${rejected-slave-promise:config-url}
master-key-upload-url = ${request-kedifa:connection-master-key-upload-url}
master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url}
kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
......@@ -284,7 +295,7 @@ software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = kedifa
name = kedifa
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url csr_id-url csr_id-certificate
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url csr_id-url csr_id-certificate monitor-base-url
{% set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in slapparameter_dict.keys() %}
......@@ -331,6 +342,7 @@ extra-context =
[monitor-conf-parameters]
monitor-url-list +=
${request-kedifa:connection-monitor-base-url}
{% for frontend in frontend_section_list %}
{{ ' ${' + frontend + ':connection-monitor-base-url}' }}
{% endfor %}
......@@ -455,6 +467,105 @@ update-command = ${:command}
{% endfor %}
{% endif %}
[rejected-slave-json]
recipe = slapos.recipe.template:jinja2
filename = rejected-slave.json
directory = ${directory:promise-output}
rendered = ${:directory}/${:filename}
template = {{ parameter_dict['template_empty'] }}
{% if rejected_slave_title_dict %}
content = {{ dumps(json_module.dumps(rejected_slave_title_dict, indent=2)) }}
{% else %}
content =
{% endif %}
context =
key content :content
[directory]
plugin = ${:etc}/plugin
service = ${:etc}/service
promise-output = ${:srv}/promise-output
[rejected-slave-publish-configuration]
ip = {{ instance_parameter['ipv6-random'] }}
port = 14455
[rejected-slave-publish]
directory = ${rejected-slave-json:directory}
url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename}
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['caddy'] }}
-conf ${rejected-slave-template:rendered}
-log stderr
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root ${:directory}
wrapper-path = ${directory:service}/rejected-slave-publish
hash-files =
${buildout:directory}/software_release/buildout.cfg
${rejected-slave-template:rendered}
${rejected-slave-certificate:certificate}
[rejected-slave-certificate]
recipe = plone.recipe.command
certificate = ${directory:etc}/rejected-slave.pem
key = ${:certificate}
stop-on-error = True
update-command = ${:command}
command =
[ -f ${:certificate} ] && find ${:certificate} -type f -mtime +3 -delete
if ! [ -f ${:certificate} ] ; then
openssl req -new -newkey rsa:2048 -sha256 -subj \
"/CN=${rejected-slave-publish-configuration:ip}" \
-days 5 -nodes -x509 -keyout ${:certificate} -out ${:certificate}
fi
[rejected-slave-password]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.rejected-slave.passwd
bytes = 8
user = admin
[rejected-slave-template]
recipe = slapos.recipe.template:jinja2
template = inline:
https://:${rejected-slave-publish-configuration:port}/ {
basicauth / ${rejected-slave-password:user} ${rejected-slave-password:passwd}
tls ${rejected-slave-certificate:certificate} ${rejected-slave-certificate:key}
bind ${rejected-slave-publish-configuration:ip}
log stderr
errors stderr
}
rendered = ${directory:etc}/Caddyfile-rejected-slave
[promise-plugin-base]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
content =
from slapos.promise.plugin.${:module} import RunPromise
output = ${directory:plugin}/${:name}
[promise-rejected-slave-publish-ip-port]
<= promise-plugin-base
module = check_port_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}
[rejected-slave-promise]
<= promise-plugin-base
module = check_port_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
config-state = empty
config-url = ${rejected-slave-publish:url}
[buildout]
extends =
{{ common_profile }}
......@@ -464,6 +575,8 @@ parts =
publish-slave-information
publish-information
request-kedifa
rejected-slave-promise
promise-rejected-slave-publish-ip-port
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
......
......@@ -11,6 +11,18 @@
"title": "Frontend Replication Quantity",
"type": "integer"
},
"apache-certificate": {
"description": "SSL Certificate used by the server. By appending to it CA certificate it is possible to use this field to replace not implemented apache-ca-certificate. Deprecated, please use master-key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate, with optional CA certificate",
"type": "string"
},
"apache-key": {
"description": "SSL Key used by the server. Deprecated, please use master-key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Key",
"type": "string"
},
"domain": {
"description": "Base Domain for create subdomains (ie.: example.com).",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
......@@ -29,7 +41,7 @@
},
"global-disable-http2": {
"default": "false",
"description": "Disables globally HTTP2 in Caddy, thus enable-http2-by-default here and enable-http2 have no effect. Rational is that in some loaded environments it is better to run Caddy without any HTTP2 capabilities.",
"description": "Disables globally HTTP2 in Caddy, thus enable-http2-by-default here and enable-http2 have no effect. Rational is that in some loaded environments it is better to run Caddy without any HTTP2 capabilities. Warning: Changing the parameter will result in restarting Caddy process.",
"enum": [
"true",
"false"
......@@ -40,7 +52,7 @@
"mpm-graceful-shutdown-timeout": {
"default": 5,
"description": "Value passed to -grace parameter of Caddy, see https://caddyserver.com/docs/cli .",
"title": "Duration of the graceful shutdown period",
"title": "Duration of the graceful shutdown period. Warning: Changing the parameter will result in restarting Caddy process.",
"type": "integer"
},
"nginx-domain": {
......@@ -61,7 +73,7 @@
},
"enable-quic": {
"default": "false",
"description": "Enables QUIC - Quick UDP Internet Connections. Note that this is experimental feature, thus can result in undefined behaviour.",
"description": "Enables QUIC - Quick UDP Internet Connections. Note that this is experimental feature, thus can result in undefined behaviour. Warning: Changing the parameter will result in restarting Caddy process.",
"enum": [
"true",
"false"
......
......@@ -7,13 +7,16 @@ extends =
{{ parameter_dict['monitor_template'] }}
parts =
monitor-base
directory
kedifa
promise-kedifa-http-reply
slave-kedifa-information
caucased
caucased-promise
caucase-updater
expose-csr_id
promise-expose-csr_id-ip-port
[caucased]
hash-files = ${buildout:directory}/software_release/buildout.cfg
......@@ -50,6 +53,7 @@ run = ${:var}/run
service = ${:etc}/service
etc-run = ${:etc}/run
promise = ${:etc}/promise
plugin = ${:etc}/plugin
# KeDiFa directories
kedifa = ${:srv}/kedifa
......@@ -151,6 +155,13 @@ template = inline:
rendered = ${directory:caddy-csr_id}/Caddyfile
[promise-expose-csr_id-ip-port]
<= promise-plugin-base
module = check_port_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
depends = ${store-csr_id:command}
recipe = slapos.cookbook:wrapper
......@@ -184,6 +195,14 @@ context =
section directory directory
${:extra-context}
[promise-plugin-base]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
content =
from slapos.promise.plugin.${:module} import RunPromise
output = ${directory:plugin}/${:name}
[kedifa-config]
ip = {{ instance_parameter['ipv6-random'] }}
port = {{ instance_parameter['configuration.kedifa_port'] }}
......@@ -200,6 +219,15 @@ recipe = slapos.cookbook:wrapper
command-line = kill -SIGHUP `cat ${kedifa-config:pidfile}`
wrapper-path = ${directory:etc-run}/kedifa-reloader
[promise-kedifa-http-reply]
<= promise-plugin-base
module = check_url_available
name = kedifa-http-reply.py
# Kedifa replies 500 on /, so use it to be sure that Kedifa replied
config-http_code = 500
config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
config-ca-cert-file = ${kedifa-config:ca-certificate}
[kedifa]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['kedifa'] }}
......@@ -222,6 +250,7 @@ hash-files = ${buildout:directory}/software_release/buildout.cfg
{%- do slave_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%}
{%- do slave_kedifa_information.__setitem__(slave_reference, slave_dict) %}
[{{ slave_reference }}-auth-random-generate]
recipe = plone.recipe.command
......
......@@ -155,6 +155,27 @@
"title": "Verify Backend Certificates",
"type": "string"
},
"ssl_crt": {
"default": "",
"description": "Content of the SSL Certificate file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate",
"type": "string"
},
"ssl_key": {
"default": "",
"description": "Content of the SSL Key file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Key",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_proxy_ca_crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
......
......@@ -45,6 +45,10 @@
"warning-list": {
"description": "List of warning found during the request.",
"type": "array"
},
"kedifa-caucase-url": {
"description": "URL to caucase used by KeDiFa",
"type": "string"
}
},
"type": "object"
......
......@@ -32,6 +32,8 @@ create = true
{% if master_key_download_url %}
{% do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
{% else %}
{% do kedifa_updater_mapping.append(('notreadyyet', master_certificate, apache_certificate)) %}
{% endif %}
{% if slave_kedifa_information %}
......@@ -48,16 +50,20 @@ content =
from slapos.promise.plugin.${:module} import RunPromise
output = {{ plugin_directory }}/${:name}
# empty section if no cached slaves are available
[slave-log-cache-direct-directory-dict]
{# Loop thought slave list to set up slaves #}
{% for slave_instance in slave_instance_list %}
{% set slave_type = slave_instance.get('type', '') %}
{% set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %}
{% set slave_reference = slave_instance.get('slave_reference') %}
{% set slave_kedifa = slave_kedifa_information.get(slave_reference) %}
{% if slave_kedifa %}
{% set key_download_url = slave_kedifa.get('key-download-url') %}
{% else %}
{% set key_download_url = '' %}
{% set key_download_url = 'notreadyyet' %}
{% endif %}
{% set slave_type = slave_instance.get('type', '') %}
{% set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference %}
{% set slave_parameter_dict = generic_instance_parameter_dict.copy() %}
{% set slave_publish_dict = {} %}
......@@ -71,6 +77,9 @@ output = {{ plugin_directory }}/${:name}
{% do part_list.extend([slave_logrotate_section, slave_section_title]) %}
{% set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
{% if enable_cache %}
{% set slave_log_cache_direct_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-cache-direct-logs" %}
{% endif %}
{# Pass HTTP2 switch #}
{% do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
......@@ -85,6 +94,12 @@ output = {{ plugin_directory }}/${:name}
{% do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
{% do slave_instance.__setitem__('access_log', slave_parameter_dict.get('access_log')) %}
{% do slave_instance.__setitem__('error_log', slave_parameter_dict.get('error_log')) %}
{% if enable_cache %}
{% do slave_parameter_dict.__setitem__('access_log_cache_direct', '/'.join([caddy_log_cache_direct_directory, '%s_access_log' % slave_reference])) %}
{% do slave_parameter_dict.__setitem__('error_log_cache_direct', '/'.join([caddy_log_cache_direct_directory, '%s_error_log' % slave_reference])) %}
{% do slave_instance.__setitem__('access_log_cache_direct', slave_parameter_dict.get('access_log_cache_direct')) %}
{% do slave_instance.__setitem__('error_log_cache_direct', slave_parameter_dict.get('error_log_cache_direct')) %}
{% endif %}
{# Add slave log directory to the slave log access dict #}
{% do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
......@@ -100,10 +115,11 @@ output = {{ plugin_directory }}/${:name}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) %}
{% endif %}
{% set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %}
{% if enable_cache and 'url' in slave_instance %}
{% if 'domain' in slave_instance %}
{% do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) %}
{% if not slave_instance.get('custom_domain') %}
{% do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) %}
{% endif %}
{% endif %}
{% do slave_instance.__setitem__('backend_url', slave_instance.get('url')) %}
{% do slave_instance.__setitem__('https_backend_url', slave_instance.get('https-url', slave_instance.get('url'))) %}
......@@ -123,22 +139,10 @@ output = {{ plugin_directory }}/${:name}
[slave-log-directory-dict]
{{slave_reference}} = {{ slave_log_folder }}
[slave-log-directories]
<= slave-log-directory-dict
recipe = slapos.cookbook:mkdirectory
{% do part_list.append('slave-log-directories') %}
[caddy-log-access]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directory-dict
section slave_password slave-password
section parameter_dict caddy-log-access-parameters
{% do part_list.append('caddy-log-access') %}
{% if enable_cache %}
[slave-log-cache-direct-directory-dict]
{{slave_reference}}_cache_direct = {{ slave_log_cache_direct_folder }}
{% endif %}
[slave-password]
{{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
......@@ -149,6 +153,13 @@ extra-context =
name = ${:_buildout_section_name_}
log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
backup = {{ slave_log_folder }}
{% if enable_cache %}
[{{slave_logrotate_section}}_cache_direct]
<= logrotate-entry-base
name = ${:_buildout_section_name_}
log = {{slave_parameter_dict.get('access_log_cache_direct')}} {{slave_parameter_dict.get('error_log_cache_direct')}}
backup = {{ slave_log_cache_direct_folder }}
{% endif %}
{# integrate current logs inside #}
[{{slave_ln_section}}]
......@@ -261,28 +272,6 @@ filename = {{ '%s.conf' % slave_reference }}
{{ '\n' }}
{% set check_error_log_section_title = 'check-%s-error-log-last-hour' % slave_instance.get('slave_reference') %}
{% do part_list.append(check_error_log_section_title) %}
[{{ check_error_log_section_title }}]
<= promise-plugin-base
module = check_error_on_http_log
name = {{ check_error_log_section_title }}.py
config-log-file = {{ slave_instance.get('error_log') }}
config-maximum-delay = 3600
# promise frequency in minutes (2 times/day)
config-frequency = 720
{% set check_error_log_section_title = 'check-%s-error-log-last-day' % slave_instance.get('slave_reference') %}
{% do part_list.append(check_error_log_section_title) %}
[{{ check_error_log_section_title }}]
<= promise-plugin-base
module = check_error_on_http_log
name = {{ check_error_log_section_title }}.py
config-log-file = {{ slave_instance.get('error_log') }}
config-maximum-delay = 86400
# promise frequency in minutes (2 times/day)
config-frequency = 720
{% set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %}
{% if monitor_ipv6_test %}
{% set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %}
......@@ -348,6 +337,29 @@ recipe = slapos.cookbook:publish
{# End of the main for loop#}
{% endfor %}
[slave-log-directories]
<= slave-log-directory-dict
recipe = slapos.cookbook:mkdirectory
{% do part_list.append('slave-log-directories') %}
[slave-log-cache-direct-directories]
<= slave-log-cache-direct-directory-dict
recipe = slapos.cookbook:mkdirectory
{% do part_list.append('slave-log-cache-direct-directories') %}
[caddy-log-access]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directory-dict
section slave_password slave-password
section parameter_dict caddy-log-access-parameters
{% do part_list.append('caddy-log-access') %}
###############################################
### Prepare virtualhost for slaves using cache
......@@ -442,6 +454,12 @@ command-line = {{ kedifa_updater }}
wrapper-path = {{ service_directory }}/kedifa-updater
hash-files = ${buildout:directory}/software_release/buildout.cfg
[kedifa-updater-run]
recipe = plone.recipe.command
stop-on-error = True
command = {{ kedifa_updater }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ frontend_graceful_reload }}"
update-command = ${:command}
[kedifa-updater-mapping]
recipe = slapos.recipe.template:jinja2
file = {{ kedifa_updater_mapping_file }}
......@@ -466,6 +484,7 @@ extends =
parts +=
kedifa-updater
kedifa-updater-run
caddy-log-access-header
{% for part in part_list %}
{{ ' %s' % part }}
......@@ -476,6 +495,7 @@ parts +=
tunnel-6to4-base-cached_port
tunnel-6to4-base-ssl_cached_port
expose-csr_id
promise-expose-csr_id-ip-port
cache-access = {{ cache_access }}
......@@ -528,6 +548,13 @@ template = inline:
rendered = {{ directory_caddy_csr_id }}/Caddyfile
[promise-expose-csr_id-ip-port]
<= promise-plugin-base
module = check_port_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
depends = ${store-csr_id:command}
recipe = slapos.cookbook:wrapper
......
......@@ -36,6 +36,13 @@
insecure_skip_verify
{%- endif %}
}
log / {{ slave_parameter.get('access_log_cache_direct') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log_cache_direct') }} {
rotate_size 0
}
}
# SSL-enabled backends
......@@ -58,4 +65,11 @@
insecure_skip_verify
{%- endif %}
}
log / {{ slave_parameter.get('access_log_cache_direct') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log_cache_direct') }} {
rotate_size 0
}
}
#!/bin/sh
set -e
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} 2> /dev/null | sort -k 66 > $NSIGNATURE_FILE
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null ; then
# No changes since last run just propagate information
rm -f ${NSIGNATURE_FILE}
exit 1
else
# Changes since last run, so store new value and propagate information
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
exit 0
fi
#!/bin/sh
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
set -e
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} | sort -k 66 > $NSIGNATURE_FILE
# If no diff, no restart for now
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null ; then
if {{ caddy_configuration_state }} ; then
echo "Reloading.."
{{ graceful_reload_command }}
else
echo "Nothing changed, so nothing to reload"
exit 0
fi
echo "Reloading.."
{{ graceful_reload_command }}
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
......@@ -48,6 +48,7 @@
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
{% endif %}
{% endfor %}
......
#!/bin/sh
set -e
LOG_DIR={{ log_dir }}
LOGROTATE_DIR={{ rotate_dir }}
PATTERN={{ pattern }}
COMPRESS={{ xz_binary }}
KEEP_DAYS={{ keep_days }}
# Move out ${PATTERN} files
find ${LOG_DIR} -maxdepth 1 -type f -name ${PATTERN} -exec mv {} ${LOGROTATE_DIR}/ \;
# Compress
find ${LOGROTATE_DIR} -maxdepth 1 -type f -name ${PATTERN} -exec ${COMPRESS} -9 {} \;
# Retent old files
find ${LOGROTATE_DIR} -maxdepth 1 -type f -mtime +${KEEP_DAYS} -delete
......@@ -5,6 +5,6 @@ log.ascii {
Format = squid,
Filename = 'squid',
RollingEnabled = 1,
RollingIntervalSec = 3600,
RollingIntervalSec = 86400,
RollingOffsetHr = 0
}
......@@ -119,7 +119,7 @@ CONFIG proxy.config.http.cache.http INT 1
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/cache.config.en.html
##############################################################################
CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 1
CONFIG proxy.config.http.normalize_ae_gzip INT 1
CONFIG proxy.config.http.normalize_ae_gzip INT 0
CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
CONFIG proxy.config.http.cache.cache_urls_that_look_dynamic INT 1
# https://docs.trafficserver.apache.org/records.config#proxy-config-http-cache-when-to-revalidate
......
......@@ -2,21 +2,19 @@
set -e
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
SIGNATURE_STATUS={{ signature_file }}.status
LAST_STATE_FILE={{ last_state_file }}
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} | sort -k 66 > $NSIGNATURE_FILE
# force validation each 2 hours
old_found=1
if [ -f $LAST_STATE_FILE ] ; then
old_found=$(find $LAST_STATE_FILE -mmin +120 | wc -l)
fi
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null; then
rm -f "$NSIGNATURE_FILE"
else
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
if [ "$old_found" -eq 1 ] || {{ caddy_configuration_state }} ; then
# do not catch errors during validation
set +e
{{ wrapper }} -validate
echo $? > $SIGNATURE_STATUS
echo $? > $LAST_STATE_FILE
set -e
fi
exit `cat $SIGNATURE_STATUS`
exit `cat $LAST_STATE_FILE`
T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
\ No newline at end of file
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
\ No newline at end of file
T-1/var/run/kedifa.pid
T-1/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/etc/monitor-promise/check-_test-error-log-last-day
T-2/etc/monitor-promise/check-_test-error-log-last-hour
\ No newline at end of file
......@@ -2,6 +2,14 @@ T-0/etc/plugin/buildout-T-0-status.py: OK
T-0/etc/plugin/check-free-disk-space.py: OK
T-0/etc/plugin/monitor-bootstrap-status.py: OK
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py: OK
T-0/etc/plugin/rejected-slave.py: OK
T-1/etc/plugin/buildout-T-1-status.py: OK
T-1/etc/plugin/check-free-disk-space.py: OK
T-1/etc/plugin/expose-csr_id-ip-port-listening.py: OK
T-1/etc/plugin/kedifa-http-reply.py: OK
T-1/etc/plugin/monitor-bootstrap-status.py: OK
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
T-2/etc/plugin/buildout-T-2-status.py: OK
T-2/etc/plugin/caddy_cached.py: OK
T-2/etc/plugin/caddy_frontend_ipv4_http.py: OK
......@@ -9,10 +17,9 @@ T-2/etc/plugin/caddy_frontend_ipv4_https.py: OK
T-2/etc/plugin/caddy_frontend_ipv6_http.py: OK
T-2/etc/plugin/caddy_frontend_ipv6_https.py: OK
T-2/etc/plugin/caddy_ssl_cached.py: OK
T-2/etc/plugin/check-_test-error-log-last-day.py: OK
T-2/etc/plugin/check-_test-error-log-last-hour.py: OK
T-2/etc/plugin/check-free-disk-space.py: OK
T-2/etc/plugin/frontend-caddy-configuration-promise.py: ERROR
T-2/etc/plugin/expose-csr_id-ip-port-listening.py: OK
T-2/etc/plugin/frontend-caddy-configuration-promise.py: OK
T-2/etc/plugin/monitor-bootstrap-status.py: OK
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
T-2/etc/plugin/re6st-connectivity.py: OK
......
T-0:bootstrap-monitor EXITED
T-0:certificate_authority-{hash}-on-watch RUNNING
T-0:crond-{hash}-on-watch RUNNING
T-0:monitor-httpd-{hash}-on-watch EXITED
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch EXITED
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash}-on-watch RUNNING
T-1:expose-csr_id-{hash}-on-watch RUNNING
T-1:kedifa-{hash}-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-2:6tunnel-11080-{hash}-on-watch STOPPED
T-2:6tunnel-11443-{hash}-on-watch STOPPED
T-2:6tunnel-26011-{hash}-on-watch STOPPED
T-2:6tunnel-26012-{hash}-on-watch STOPPED
T-1:monitor-httpd-{hash-generic}-on-watch EXITED
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch STOPPED
T-2:6tunnel-11443-{hash-generic}-on-watch STOPPED
T-2:6tunnel-26011-{hash-generic}-on-watch STOPPED
T-2:6tunnel-26012-{hash-generic}-on-watch STOPPED
T-2:bootstrap-monitor STOPPED
T-2:certificate_authority-{hash}-on-watch STOPPED
T-2:crond-{hash}-on-watch STOPPED
T-2:expose-csr_id-{hash}-on-watch STOPPED
T-2:certificate_authority-{hash-generic}-on-watch STOPPED
T-2:crond-{hash-generic}-on-watch STOPPED
T-2:expose-csr_id-{hash-generic}-on-watch STOPPED
T-2:frontend-caddy-safe-graceful STOPPED
T-2:frontend_caddy-{hash}-on-watch STOPPED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch STOPPED
T-2:kedifa-login-certificate-caucase-updater-on-watch STOPPED
T-2:kedifa-updater-{hash}-on-watch STOPPED
T-2:monitor-httpd-{hash}-on-watch STOPPED
T-2:kedifa-updater-{hash-generic}-on-watch STOPPED
T-2:monitor-httpd-{hash-generic}-on-watch STOPPED
T-2:monitor-httpd-graceful STOPPED
T-2:trafficserver-{hash}-on-watch STOPPED
T-2:trafficserver-{hash-generic}-on-watch STOPPED
T-2:trafficserver-reload STOPPED
watchdog:watchdog RUNNING
\ No newline at end of file
T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
\ No newline at end of file
T-0/var/run/monitor-httpd.pid
T-0/var/run/monitor/monitor-bootstrap.pid
T-1/var/run/kedifa.pid
T-2/var/run/caddy_graceful_signature
T-2/var/run/caddy_validate_signature
T-2/var/run/caddy_validate_signature.status
T-1/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/caddy_configuration_last_state
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file
T-2/var/run/monitor/monitor-bootstrap.pid
T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
T-2/etc/monitor-promise/check-_site_2-error-log-last-day
T-2/etc/monitor-promise/check-_site_2-error-log-last-hour
\ No newline at end of file
......@@ -2,6 +2,14 @@ T-0/etc/plugin/buildout-T-0-status.py: OK
T-0/etc/plugin/check-free-disk-space.py: OK
T-0/etc/plugin/monitor-bootstrap-status.py: OK
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py: OK
T-0/etc/plugin/rejected-slave.py: ERROR