pax_global_header 0000666 0000000 0000000 00000000064 13712426147 0014521 g ustar 00root root 0000000 0000000 52 comment=c6cc45ab7b5d52c653832697d6e0323159c12837
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/ 0000775 0000000 0000000 00000000000 13712426147 0021361 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ 0000775 0000000 0000000 00000000000 13712426147 0023213 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/agent/ 0000775 0000000 0000000 00000000000 13712426147 0024311 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/agent/agent.cfg.in 0000664 0000000 0000000 00000000412 13712426147 0026472 0 ustar 00root root 0000000 0000000 [agent]
timeout = ${slap-parameter:timeout}
node_title = ${slap-parameter:node_title}
project_title = ${slap-parameter:project_title}
task_count = ${slap-parameter:task_count}
report_url = ${slap-parameter:report_url}
working_directory = ${directory:testnode}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/agent/instance-agent.cfg 0000664 0000000 0000000 00000001736 13712426147 0027701 0 ustar 00root root 0000000 0000000 [buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
service = $${buildout:directory}/etc/service
run = $${buildout:directory}/etc/run
agentlog = $${buildout:directory}/var/log/agent
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
testnode = $${buildout:directory}/srv/testnode
[instance]
recipe = slapos.cookbook:wrapper
command-line =
${buildout:bin-directory}/agent
--pidfile=$${buildout:directory}/srv/agent.pid
--log=$${buildout:directory}/var/log/agent/agent.log
$${agent-cfg:output}
wrapper-path = $${directory:service}/agent
output = $${:wrapper-path}
[agent-cfg]
recipe = slapos.recipe.template
url = ${agent.cfg.in:target}
output = $${directory:etc}/agent.cfg
[slap-parameter]
timeout = 3600
node_title =
test_title =
project_title =
task_count = 1
report_url =
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/agent/instance.cfg 0000664 0000000 0000000 00000000401 13712426147 0026571 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-agent:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/agent/software.cfg 0000664 0000000 0000000 00000002032 13712426147 0026621 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/lxml-python/buildout.cfg
../../component/git/buildout.cfg
../../component/pycurl/buildout.cfg
../../stack/slapos.cfg
parts =
agent.cfg.in
template
template-agent
slapos-cookbook
script
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = bcd3b3cb8a305c83bb048d5ac1c583fe
mode = 0644
[template-agent]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-agent.cfg
output = ${buildout:directory}/template-agent.cfg
md5sum = 797e80e43bbf9c0d0e0c5db88a091667
mode = 0644
[agent.cfg.in]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/agent.cfg.in
md5sum = 056b1bb005080c4324cd9d755d254131
mode = 0644
[script]
recipe = zc.recipe.egg
eggs =
${pycurl:egg}
zc.buildout
slapos.core
slapos.toolbox
erp5.util
[versions]
ecdsa = 0.13
gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.4
smmap = 0.9.0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/ 0000775 0000000 0000000 00000000000 13712426147 0025362 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/buildout.hash.cfg 0000664 0000000 0000000 00000002235 13712426147 0030616 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance]
filename = instance.cfg
md5sum = 1fbb72e4305ded9614ee80c0ee17b672
[instance-apache]
filename = instance-apachephp.cfg
md5sum = a1ddbb63ffa0041b27eaea6df85cbd00
[website1-template]
filename = templates/index.html.in
md5sum = c5695762361b801c284ee23a150cd1f1
[website2-template]
filename = templates/index.html.in
md5sum = c5695762361b801c284ee23a150cd1f1
[template-httpd-conf-hash]
filename = templates/apache.conf.in
md5sum = 4fc3f853abf702aeb6653942d4fb85d8
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/instance-apachephp.cfg 0000664 0000000 0000000 00000023142 13712426147 0031600 0 ustar 00root root 0000000 0000000 [buildout]
parts =
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
promise
frontend-ajaxupload-promise
frontend-website2-promise
frontend-website-promise
# content-promise
publish-connection-informations
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
#----------------
#--
#-- Creation of all needed directories.
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/service
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
[directory]
recipe = slapos.cookbook:mkdirectory
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
cronstamps = $${rootdirectory:etc}/cronstamps
ca-dir = $${rootdirectory:srv}/ssl
httpd-log = $${basedirectory:log}/apache
php-ini-dir = $${rootdirectory:etc}/php
tmp-php = $${rootdirectory:tmp}/php
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
report = $${rootdirectory:etc}/report
stunnel-conf = $${rootdirectory:etc}/stunnel
xml-report = $${rootdirectory:var}/xml_report
www = $${rootdirectory:srv}/www/
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests
private = $${directory:ca-dir}/private
certs = $${directory:ca-dir}/certs
newcerts = $${directory:ca-dir}/newcerts
crl = $${directory:ca-dir}/crl
#----------------
#--
#-- Deploy cron.
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
#----------------
#--
#-- Deploy logrotate.
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
#----------------
#--
#-- Deploy stunnel.
[stunnel]
recipe = slapos.cookbook:stunnel
client = true
stunnel-binary = ${stunnel:location}/bin/stunnel
remote-host = $${mariadb-urlparse:host}
remote-port = $${mariadb-urlparse:port}
local-host = $${slap-network-information:local-ipv4}
local-port = 3306
log-file = $${basedirectory:log}/stunnel.log
config-file = $${directory:stunnel-conf}/stunnel.conf
key-file = $${directory:stunnel-conf}/stunnel.key
cert-file = $${directory:stunnel-conf}/stunnel.crt
pid-file = $${basedirectory:run}/stunnel.pid
wrapper = $${rootdirectory:bin}/raw_stunnel
post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
[logrotate-entry-stunnel]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = stunnel
log = $${stunnel:log-file}
frequency = daily
rotate-num = 30
notifempty = true
create = true
post = $${stunnel:post-rotate-script}
#----------------
#--
#-- Certificate stuff.
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${basedirectory:services}/ca
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[ca-stunnel]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
executable = $${stunnel:wrapper}
wrapper = $${basedirectory:services}/stunnel
key-file = $${stunnel:key-file}
cert-file = $${stunnel:cert-file}
#----------------
#--
#-- Request MariaDB instance and parse its URL.
[request-mariadb]
<= slap-connection
recipe = slapos.cookbook:request
name = MariaDB
software-url = $${slap-connection:software-release-url}
software-type = mariadb
return = url
sla-computer_guid = $${slap-connection:computer-id}
[mariadb-urlparse]
recipe = slapos.cookbook:urlparse
url = $${request-mariadb:connection-url}
#----------------
#--
#-- Deploy Apache + PHP application.
[httpd-conf]
recipe = slapos.recipe.template
url = ${template-httpd-conf:location}/${template-httpd-conf:filename}
output = $${rootdirectory:etc}/apache.conf
mode = 0600
document_root = $${rootdirectory:srv}/www/
pid_file = $${basedirectory:run}/apache.pid
lock_file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6}
port = 8080
port2 = 8070
port3 = 8090
error_log = $${directory:httpd-log}/error.log
access_log = $${directory:httpd-log}/access.log
php_ini_dir = $${directory:php-ini-dir}
# Deploy Apache + PHP application
[apache-php]
recipe = slapos.cookbook:apachephp
source = ${application:location}
template =
configuration =
htdocs = $${directory:www}
pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock
ip = $${httpd-conf:ip}
port = $${httpd-conf:port}
url = http://[$${:ip}]:$${:port}/
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
php-ini-dir = $${directory:php-ini-dir}
tmp-dir = $${directory:tmp-php}
wrapper = $${basedirectory:services}/apache
httpd-binary = ${apache:location}/bin/httpd
default-conf = false
httpd-conf = $${httpd-conf:output}
mysql-username = $${mariadb-urlparse:username}
mysql-password = $${mariadb-urlparse:password}
mysql-database = $${mariadb-urlparse:path}
mysql-host = $${stunnel:local-host}
mysql-port = $${stunnel:local-port}
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-php:error-log} $${apache-php:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
#----------------
#--
#-- Request frontends.
[request-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Frontend-Website
# XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url}
slave = true
config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
return = site_url
config-custom_domain = $${slap-parameter:domain}
[request-frontend-ajaxupload]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Frontend-FileManager
# XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url}
slave = true
config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port2}/
return = site_url
config-custom_domain = $${slap-parameter:domain2}
[request-frontend2]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Frontend-Website2
# XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url}
slave = true
config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port3}/
return = site_url
config-custom_domain = $${slap-parameter:domain3}
#----------------
#--
#-- Publish instance parameters.
[publish-connection-informations]
recipe = slapos.cookbook:publish
website-backend-url = $${apache-php:url}
website-url = $${request-frontend:connection-site_url}
website2-backend-url = http://[$${apache-php:ip}]:$${httpd-conf:port3}
website2-url = $${request-frontend2:connection-site_url}
filemanager-backend-url = http://[$${apache-php:ip}]:$${httpd-conf:port2}
fileManager-url = $${request-frontend-ajaxupload:connection-site_url}
mysql-username = $${mariadb-urlparse:username}
mysql-password = $${mariadb-urlparse:password}
mysql-database = $${mariadb-urlparse:path}
mysql-host = $${stunnel:local-host}
mysql-port = $${stunnel:local-port}
#----------------
#--
#-- Deploy promises scripts.
[promise]
recipe = slapos.cookbook:check_port_listening
path = $${basedirectory:promises}/apache
hostname = $${apache-php:ip}
port = $${apache-php:port}
[frontend-website-promise]
recipe = slapos.cookbook:check_url_available
path = $${basedirectory:promises}/frontend-website
url = $${request-frontend:connection-site_url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[frontend-website2-promise]
recipe = slapos.cookbook:check_url_available
path = $${basedirectory:promises}/frontend-website2
url = $${request-frontend2:connection-site_url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[frontend-ajaxupload-promise]
recipe = slapos.cookbook:check_url_available
path = $${basedirectory:promises}/frontend-ajaxupload
url = $${request-frontend-ajaxupload:connection-site_url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[content-promise]
recipe = slapos.cookbook:check_page_content
path = $${basedirectory:promises}/content
url = $${request-frontend-ajaxupload:connection-site_url}
match = AjaXplorer
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
[slap-parameter]
# Default value if no domain is specified
domain =
domain2 =
domain3 =
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
# Default value if no ssh parameter is specified
logbox-ip =
logbox-port =
logbox-user =
logbox-passwd =
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/instance.cfg 0000664 0000000 0000000 00000000447 13712426147 0027654 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-apache:output}
mariadb = ${instance-mariadb:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/software.cfg 0000664 0000000 0000000 00000004415 13712426147 0027701 0 ustar 00root root 0000000 0000000 [buildout]
parts =
slapos-cookbook
subversion
apache-php
instance
instance-apache
instance-mariadb
template-httpd-conf
application
website1-template
website2-template
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/apache/buildout.cfg
../../component/apache-php/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/subversion/buildout.cfg
../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg
../../component/dash/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/lxml-python/buildout.cfg
[instance]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[instance-apache]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-apachephp.cfg
mode = 0644
[instance-mariadb]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/../../stack/lamp/mariadb/instance-mariadb.cfg.in
output = ${buildout:directory}/template-mariadb.cfg
mode = 0644
[application]
recipe = hexagonit.recipe.download
ignore-existing = true
url = http://garr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/dev-channel/4.3.4/ajaxplorer-core-4.3.4.tar.gz
md5sum = 2f2ff8bda7bbe841ef0e870c724eb74f
strip-top-level-dir = true
[website]
recipe = z3c.recipe.mkdir
path1 = ${application:location}/data/files/website
path2 = ${application:location}/data/files/website2
paths =
${:path1}
${:path2}
[website1-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${website:path1}/index.html
mode = 0644
[website2-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${website:path2}/index.html
mode = 0644
[template-httpd-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${template-httpd-conf-hash:filename}
mode = 0644
filename = apache.conf.in
md5sum = ${template-httpd-conf-hash:md5sum}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[versions]
# Numpy 1.7.0 doesn't install well
numpy = 1.16.4
# websockify 0.4.1 doesn't install well
websockify = 0.3.0
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
z3c.recipe.mkdir = 0.5
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/templates/ 0000775 0000000 0000000 00000000000 13712426147 0027360 5 ustar 00root root 0000000 0000000 apache.conf.in 0000664 0000000 0000000 00000005620 13712426147 0032001 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/templates # Apache static configuration
# Automatically generated
# Basic server configuration
PidFile "${:pid_file}"
Listen ${:ip}:8070
Listen ${:ip}:8080
Listen ${:ip}:8090
PHPINIDir ${:php_ini_dir}
ServerAdmin someone@email
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
AddType application/x-httpd-php .php .phtml .php5 .php4
AddType application/x-httpd-php-source .phps
# Log configuration
ErrorLog "${:error_log}"
LogLevel warn
LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b" common
CustomLog "${:access_log}" common
NameVirtualHost ${:ip}:8090
NameVirtualHost ${:ip}:8080
NameVirtualHost ${:ip}:8070
#ServerName www.example.com
# Directory protection
Options FollowSymLinks
AllowOverride None
Require all denied
Options FollowSymLinks
AllowOverride All
Require all granted
DocumentRoot ${:document_root}
DirectoryIndex index.html index.php
#ServerName www.example.com
# Directory protection
Options FollowSymLinks
AllowOverride None
Require all denied
Options FollowSymLinks
AllowOverride All
Require all granted
DocumentRoot ${:document_root}data/files/website/
DirectoryIndex index.html index.php
#ServerName www.example.com
# Directory protection
Options FollowSymLinks
AllowOverride None
Require all denied
Options FollowSymLinks
AllowOverride All
Require all granted
DocumentRoot ${:document_root}data/files/website2/
DirectoryIndex index.html index.php
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule dir_module modules/mod_dir.so
LoadModule php5_module modules/libphp5.so
LoadModule alias_module modules/mod_alias.so
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/ajaxplorer/templates/index.html.in0000664 0000000 0000000 00000000537 13712426147 0031767 0 ustar 00root root 0000000 0000000
New Document - Created By AjaXplorer
Hi, this website folder is still empty. Please go to your File Manager administration and replace this folder by your new website files.
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/ 0000775 0000000 0000000 00000000000 13712426147 0026251 5 ustar 00root root 0000000 0000000 README.apache_frontend.rst 0000664 0000000 0000000 00000040403 13712426147 0033001 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend apache_frontend
===============
Frontend system using Apache, allowing to rewrite and proxy URLs like
myinstance.myfrontenddomainname.com to real IP/URL of myinstance.
apache_frontend works using the master instance / slave instance design.
It means that a single main instance of Apache will be used to act as frontend
for many slaves.
Software type
=============
Apache frontend is available in 3 software types:
* default : The standard way to use the apache frontend configuring everything with a few given parameters
* custom-personal : This software type allow each slave to edit its apache configuration file
About frontend replication
===========================
Slaves of the root instance are sent as a parameter to requested frontends which will process them. The only difference is that they will then return the « would-be published information » to the root instance instead of publishing it. The root instance will then do a synthesis and publish the information to its slaves. The replicate instance only use 5 type of parameters for itself and will transmit the rest to requested frontends.
These parameters are :
* "-frontend-type" : the type to deploy frontends with. (default to 2)
* "-frontend-quantity" : The quantity of frontends to request (default to "default")
* "-frontend-i-state": The state of frontend i
* "-frontend-config-i-foo": Frontend i will be requested with parameter foo
* "-frontend-software-release-url": Software release to be used for frontends, default to the current software release
* "-sla-i-foo" : where "i" is the number of the concerned frontend (between 1 and "-frontend-quantity") and "foo" a sla parameter.
ex:
3
custom-personal
stopped
COMP-1234
http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.183:/software/apache-frontend/software.cfg
will request the third frontend on COMP-1234. All frontends will be of software type "custom-personal". The second frontend will be requested with the state stopped
Note: the way slaves are transformed to a parameter avoid modifying more than 3 lines in the frontend logic.
Important NOTE: The way you ask for slave to a replicate frontend is the same as the one you would use for the software given in "-frontend-quantity". Do not forget to use "replicate" for software type. XXXXX So far it is not possible to do a simple request on a replicate frontend if you do not know the software_guid or other sla-parameter of the master instance. In fact we do not know yet the software type of the "requested" frontends. TO BE IMPLEMENTED
XXX Should be moved to specific JSON File
Extra-parameter per frontend with default :
ram-cache-size = 1G
disk-cache-size = 8G
How to deploy a frontend server
===============================
This is to deploy an entire frontend server with a public IPv4.
If you want to use an already deployed frontend to make your service available
via ipv4, switch to the "Example" parts.
First, you will need to request a "master" instance of Apache Frontend with:
* A "domain" parameter where the frontend will be available
* A "public-ipv4" parameter to state which public IPv4 will be used
like::
moulefrite.org
xxx.xxx.xxx.xxx
Then, it is possible to request many slave instances
(currently only from slapconsole, UI doesn't work yet)
of Apache Frontend, like::
instance = request(
software_release=apache_frontend,
partition_reference='frontend2',
shared=True,
partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
)
Those slave instances will be redirected to the "master" instance,
and you will see on the "master" instance the associated RewriteRules of
all slave instances.
Finally, the slave instance will be accessible from:
https://someidentifier.moulefrite.org.
About SSL
=========
Default and custom-personal software type can handle specific ssl for one slave instance.
IMPORTANT: One apache can not serve more than One specific SSL VirtualHost and be compatible with obsolete browser (i.e.: IE8). See http://wiki.apache.org/httpd/NameBasedSSLVHostsWithSNI
#How to have custom configuration in frontend server
#===================================================
#
#In your instance directory, you, as sysadmin, can directly edit two
#configuration files that won't be overwritten by SlapOS to customize your
#instance:
#
# * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.custom.conf
# * $PARTITION_PATH/srv/srv/apache-conf.d/apache_frontend.virtualhost.custom.conf
#
#The first one is included in the end of the main apache configuration file.
#The second one is included in the virtualhost of the main apache configuration file.
#
#SlapOS will jsut create those two files for you, then completely forget them.
#
#Note: make sure that the UNIX user of the instance has read access to those
#files if you edit them.
Instance Parameters
===================
Master Instance Parameters
--------------------------
The parameters for instances are described at `instance-apache-input-schema.json `_.
Here some additional informations about the parameters listed, below:
domain
~~~~~~
name of the domain to be used (example: mydomain.com). Subdomains of this
domain will be used for the slave instances (example:
instance12345.mydomain.com). It is then recommended to add a wildcard in DNS
for the subdomains of the chosen domain like::
*.mydomain.com. IN A 123.123.123.123
Using the IP given by the Master Instance.
"domain" is a mandatory Parameter.
public-ipv4
~~~~~~~~~~~
Public ipv4 of the frontend (the one Apache will be indirectly listening to)
port
~~~~
Port used by Apache. Optional parameter, defaults to 4443.
plain_http_port
~~~~~~~~~~~~~~~
Port used by apache to serve plain http (only used to redirect to https).
Optional parameter, defaults to 8080.
Slave Instance Parameters
-------------------------
The parameters for instances are described at `instance-slave-apache-input-schema.json `_.
Here some additional informations about the parameters listed, below:
path
~~~~
Only used if type is "zope".
Will append the specified path to the "VirtualHostRoot" of the zope's
VirtualHostMonster.
"path" is an optional parameter, ignored if not specified.
Example of value: "/erp5/web_site_module/hosting/"
apache_custom_https
~~~~~~~~~~~~~~~~~~~
Raw apache configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the https port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above.
NOTE: If you want to use the cache, use the apache option "ProxyPreserveHost On"
apache_custom_http
~~~~~~~~~~~~~~~~~~
Raw apache configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the http port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above
NOTE: If you want to use the cache, use the apache option "ProxyPreserveHost On"
url
~~~
Necesarry to activate cache. url of backend to use.
"url" is an optional parameter.
Example: http://mybackend.com/myresource
domain
~~~~~~
Necesarry to activate cache. The frontend will be accessible from this domain.
"domain" is an optional parameter.
Example: www.mycustomdomain.com
enable_cache
~~~~~~~~~~~~
Necesarry to activate cache.
"enable_cache" is an optional parameter.
ssl_key, ssl_crt, ssl_ca_crt
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
SSL certificates of the slave.
They are optional.
Functionalities for apache configuration:
In the slave apache configuration you can use parameters that will be replaced during instanciation. They should be entered as python templates parameters ex:" %(parameter)s"
* cache_access : url of the cache. Should replace backend url in configuration to use the cache
* error_log : path of the slave error log in order to log in a deferenciated file.
* error_log : path of the slave access log in order to log in a deferenciated file.
* ssl_key, ssl_crt, ssl_ca_crt, ssl_crs : path of the certificates given in slave instance parameters
Examples
========
Here are some example of how to make your SlapOS service available through
an already deployed frontend.
Simple Example (default)
------------------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
redirected and accessible from the proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
}
)
Zope Example (default)
----------------------
Request slave frontend instance using a Zope backend so that
https://[1:2:3:4:5:6:7:8]:1234 will be redirected and accessible from the
proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"type":"zope",
}
)
Advanced example
-----------------
Request slave frontend instance using a Zope backend, with Varnish activated,
listening to a custom domain and redirecting to /erp5/ so that
https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
the proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"enable_cache":"true",
"type":"zope",
"path":"/erp5",
"domain":"mycustomdomain.com",
}
)
Simple Example
---------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"apache_custom_https":'
ServerName www.example.org
ServerAlias example.org
ServerAdmin geronimo@example.org
SSLEngine on
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
RewriteRule ^/(.*) https://[1:2:3:4:5:6:7:8]:1234/$1 [L,P]',
"apache_custom_http":'
ServerName www.example.org
ServerAlias www.example.org
ServerAlias example.org
ServerAdmin geronimo@example.org
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
# Not using HTTPS? Ask that guy over there.
# Dummy redirection to https. Note: will work only if https listens
# on standard port (443).
RewriteRule ^/(.*) https://[1:2:3:4:5:6:7:8]:1234/$1 [L,P],
}
)
Simple Cache Example
--------------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"domain": "www.example.org",
"enable_cache": "True",
"apache_custom_https":'
ServerName www.example.org
ServerAlias www.example.org
ServerAlias example.org
ServerAdmin geronimo@example.org
SSLEngine on
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
RewriteRule ^/(.*) %(cache_access)s/$1 [L,P]',
"apache_custom_http":'
ServerName www.example.org
ServerAlias www.example.org
ServerAlias example.org
ServerAdmin geronimo@example.org
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
# Not using HTTPS? Ask that guy over there.
# Dummy redirection to https. Note: will work only if https listens
# on standard port (443).
RewriteRule ^/(.*) %(cache_access)s/$1 [L,P],
}
)
Advanced example
-----------------
Request slave frontend instance using custom apache configuration, willing to use cache and ssl certificates.
listening to a custom domain and redirecting to /erp5/ so that
https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
the proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"enable_cache":"true",
"type":"zope",
"path":"/erp5",
"domain":"example.org",
"apache_custom_https":'
ServerName www.example.org
ServerAlias www.example.org
ServerAdmin example.org
SSLEngine on
SSLProxyEngine on
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
# Use personal ssl certificates
SSLCertificateFile %(ssl_crt)s
SSLCertificateKeyFile %(ssl_key)s
SSLCACertificateFile %(ssl_ca_crt)s
SSLCertificateChainFile %(ssl_ca_crt)s
# Configure personal logs
ErrorLog "%(error_log)s"
LogLevel info
LogFormat "%%h %%l %%{REMOTE_USER}i %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
CustomLog "%(access_log)s" combined
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
# Redirect / to /index.html
RewriteRule ^/$ /index.html [R=302,L]
# Use cache
RewriteRule ^/(.*) %(cache_access)s/VirtualHostBase/https/www.example.org:443/erp5/VirtualHostRoot/$1 [L,P]',
"apache_custom_http":'
ServerName www.example.org
ServerAlias www.example.org
ServerAlias example.org
ServerAdmin geronimo@example.org
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
# Configure personal logs
ErrorLog "%(error_log)s"
LogLevel info
LogFormat "%%h %%l %%{REMOTE_USER}i %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
CustomLog "%(access_log)s" combined
# Not using HTTPS? Ask that guy over there.
# Dummy redirection to https. Note: will work only if https listens
# on standard port (443).
RewriteRule ^/(.*)$ https://%%{SERVER_NAME}%%{REQUEST_URI}',
"ssl_key":"-----BEGIN RSA PRIVATE KEY-----
XXXXXXX..........XXXXXXXXXXXXXXX
-----END RSA PRIVATE KEY-----",
"ssl_crt":'-----BEGIN CERTIFICATE-----
XXXXXXXXXXX.............XXXXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----',
"ssl_ca_crt":'-----BEGIN CERTIFICATE-----
XXXXXXXXX...........XXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----',
"ssl_csr":'-----BEGIN CERTIFICATE REQUEST-----
XXXXXXXXXXXXXXX.............XXXXXXXXXXXXXXXXXX
-----END CERTIFICATE REQUEST-----',
}
)
Notes
=====
It is not possible with slapos to listen to port <= 1024, because process are
not run as root.
Solution 1 (IPv4 only)
----------------------
It is a good idea then to go on the node where the instance is
and set some iptables rules like (if using default ports)::
iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 443 -j DNAT --to-destination {listening_ipv4}:4443
iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 80 -j DNAT --to-destination {listening_ipv4}:8080
Where {public ip} is the public IP of your server, or at least the LAN IP to where your NAT will forward to.
{listening ip} is the private ipv4 (like 10.0.34.123) that the instance is using and sending as connection parameter.
Solution 2 (IPv6 only)
----------------------
It is also possible to directly allow the service to listen on 80 and 443 ports using the following command:
setcap 'cap_net_bind_service=+ep' /opt/slapgrid/$APACHE_FRONTEND_SOFTWARE_RELEASE_MD5/parts/apache/bin/httpd
Then specify in the instance parameters "port" and "plain_http_port" to be 443 and 80, respectively.
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/TODO.rst 0000664 0000000 0000000 00000000137 13712426147 0027551 0 ustar 00root root 0000000 0000000 Apache:
=======
* (Minor Bug) Check if slave in slave_list before publishing information on it
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/buildout.hash.cfg 0000664 0000000 0000000 00000006411 13712426147 0031505 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 4adae801babadd9aed6112c1ff209076
[template-apache-frontend]
filename = instance-apache-frontend.cfg
md5sum = ec825cba3ea775452716f1da7cd2e549
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 9e76028df7e93d3e32982884d5dc0913
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
md5sum = b3d5a3573fa381fec954ad5cbf1575a0
[template-replicate-publish-slave-information]
filename = templates/replicate-publish-slave-information.cfg.in
md5sum = 7c0025a35dc62eb3cbfd07a6b04d659e
[template-apache-frontend-configuration]
filename = templates/apache.conf.in
md5sum = b666d7c4a5c1fd8020713aa53b44a386
[template-custom-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 0fda1ba57fb0c4f1b51a6a94a2bd91d3
[template-not-found-html]
filename = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-virtualhost]
filename = templates/000.conf.in
md5sum = 5fcc11ea7f3eae38b0135b94dfc42dd5
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = 7890469ecc5e7f46be5b2c3074a09b52
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
md5sum = f97dfcfae80527e242f285c9ebf11b27
[template-log-access]
filename = templates/template-log-access.conf.in
md5sum = a20683faba37b5b9c035783e811dd88d
[template-empty]
filename = templates/empty.in
md5sum = 7155b18edfe128825b8d1f48071454a6
[template-wrapper]
filename = templates/wrapper.in
md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
filename = templates/trafficserver/records.config.jinja2
md5sum = 84baef0a49c9a65e8f2d2ffdb8c1d39c
[template-trafficserver-storage-config]
filename = templates/trafficserver/storage.config.jinja2
md5sum = 117238225b3fc3c5b5be381815f44c67
[template-nginx-configuration]
filename = templates/nginx.cfg.in
md5sum = f334ef32234771aee69c06f843da1980
[template-nginx-eventsource-slave-virtualhost]
filename = templates/nginx-eventsource-slave.conf.in
md5sum = a5186f666acb2f040ede04c91e60408f
[template-nginx-notebook-slave-virtualhost]
filename = templates/nginx-notebook-slave.conf.in
md5sum = 82d74a7f2aceb2b4a7acc6259291b7f2
[template-apache-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in
md5sum = 44e235d0451932f1232f9d9e0ecc59f9
[template-apache-graceful-script]
filename = templates/apache-graceful-script.sh.in
md5sum = 41299cc64200e7b8217fb9dec20bb8b9
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/common.cfg 0000664 0000000 0000000 00000012445 13712426147 0030230 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/git/buildout.cfg
../../component/dash/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/apache/buildout.cfg
../../component/gzip/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/trafficserver/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/nodejs.cfg
# Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg
parts +=
template
template-apache-frontend
template-apache-replicate
apache
apache-antiloris
stunnel
dcron
logrotate
rdiff-backup
nginx-push-stream-output
npm-modules
proxy-by-url
http-proxy
[apache]
# install apache not shared, so that we can install antiloris modules
shared = true
[extra-eggs]
recipe = zc.recipe.egg
eggs =
websockify
erp5.util
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
mode = 0644
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/${:filename}
mode = 640
[template-slave-list]
<=download-template
filename = apache-custom-slave-list.cfg.in
[template-slave-configuration]
<=download-template
filename = custom-virtualhost.conf.in
[template-replicate-publish-slave-information]
<=download-template
filename = replicate-publish-slave-information.cfg.in
[template-apache-frontend-configuration]
<=download-template
filename = apache.conf.in
[template-custom-slave-list]
<=download-template
filename = apache-default-slave-list.cfg.in
[template-not-found-html]
<=download-template
filename = notfound.html
[template-default-virtualhost]
<=download-template
filename = 000.conf.in
[template-default-slave-virtualhost]
<=download-template
filename = default-virtualhost.conf.in
[template-cached-slave-virtualhost]
<=download-template
filename = cached-virtualhost.conf.in
[template-log-access]
<=download-template
filename = template-log-access.conf.in
[template-empty]
<=download-template
filename = empty.in
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = records.config.jinja2
download-only = true
mode = 0644
[template-trafficserver-storage-config]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = storage.config.jinja2
download-only = true
mode = 0644
# NGINX Configuration
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/nginx.cfg.in
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-apache-lazy-script-call]
<=download-template
filename = apache-lazy-script-call.sh.in
[template-apache-graceful-script]
<=download-template
filename = apache-graceful-script.sh.in
[template-nginx-eventsource-slave-virtualhost]
<=download-template
filename = nginx-eventsource-slave.conf.in
[template-nginx-notebook-slave-virtualhost]
<=download-template
filename = nginx-notebook-slave.conf.in
# Migrated from KVM recipe
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
recipe = slapos.recipe.build:download-unpacked
# use upstream when merged
url = https://lab.nexedi.com/nexedi/node-http-proxy/repository/archive.zip?ref=a5d3aff428ee8d840068b439f6ce121077f1144f
md5sum = 65602466066444c718215de41f546585
[proxy-by-url]
# https://github.com/dominictarr/proxy-by-url
recipe = slapos.recipe.build:download-unpacked
# use upstream when merged
url = https://lab.nexedi.com/nexedi/proxy-by-url/repository/archive.zip?ref=59fcb11a3e00c45b4b0362e76f29653abb313072
md5sum = c383e0c5ff31b56f7987466e8304c941
[npm-modules]
recipe = plone.recipe.command
destination = ${buildout:parts-directory}/${:_buildout_section_name_}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command =
export HOME=${:location};
rm -fr ${:destination} &&
mkdir -p ${:destination} &&
cd ${:destination} &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/development.cfg 0000664 0000000 0000000 00000001023 13712426147 0031250 0 ustar 00root root 0000000 0000000 # Development profile of apache-frontend.
# Exactly the same as software.cfg, but fetch the slapos.cookbook
# from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
# You'll need to run buildout twice for this profile.
[buildout]
extends =
# Extend in this order, otherwise "parts" will be taken from git profile
common.cfg
parts +=
slapos.toolbox-dev
[slapos.toolbox-dev]
recipe = zc.recipe.egg:develop
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
instance-apache-frontend.cfg 0000664 0000000 0000000 00000070627 13712426147 0033527 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend [buildout]
parts =
directory
configtest
logrotate
cron
cron-entry-logrotate
ca-frontend
certificate-authority
logrotate-entry-apache
logrotate-entry-nginx
apache-frontend
switch-apache-softwaretype
frontend-apache-graceful
frontend-nginx-graceful
dynamic-template-default-vh
not-found-html
port-redirection
promise-frontend-apache-configuration
promise-apache-frontend-v4-https
promise-apache-frontend-v4-http
promise-apache-frontend-v6-https
promise-apache-frontend-v6-http
promise-apache-frontend-cached
promise-apache-frontend-ssl-cached
promise-apache-is-process-older-than-dependency-set
promise-apache-mpm-watchdog
promise-nginx-frontend-v4-https
promise-nginx-frontend-v4-http
promise-nginx-frontend-v6-https
promise-nginx-frontend-v6-http
promise-nginx-configuration
promise-nginx-is-process-older-than-dependency-set
trafficserver-launcher
trafficserver-reload
trafficserver-configuration-directory
trafficserver-records-config
trafficserver-remap-config
trafficserver-plugin-config
trafficserver-storage-config
trafficserver-promise-listen-port
trafficserver-promise-cache-availability
## Nginx
nginx-frontend
## Monitor for apache
monitor-base
monitor-ats-cache-stats-wrapper
monitor-traffic-summary-last-stats-wrapper
monitor-apache-server-status-wrapper
monitor-verify-re6st-connectivity
extends = ${monitor-template:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin/
etc = $${buildout:directory}/etc/
srv = $${buildout:directory}/srv/
var = $${buildout:directory}/var/
template = $${buildout:directory}/template/
backup = $${:srv}/backup
log = $${:var}/log
run = $${:var}/run
service = $${:etc}/service
etc-run = $${:etc}/run
promise = $${:etc}/promise
logrotate-backup = $${:backup}/logrotate
logrotate-entries = $${:etc}/logrotate.d
cron-entries = $${:etc}/cron.d
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
ca-dir = $${:srv}/ssl
varnginx = $${:var}/nginx
[switch-apache-softwaretype]
recipe = slapos.cookbook:softwaretype
single-default = $${dynamic-custom-personal-template-slave-list:rendered}
single-custom-personal = $${dynamic-custom-personal-template-slave-list:rendered}
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Define default parameter(s) that will be used later, in case user didn't
# specify it
# All parameters are available through the configuration.XX syntax.
# All possible parameters should have a default.
configuration.domain = example.org
configuration.public-ipv4 =
configuration.port = 4443
configuration.plain_http_port = 8080
configuration.plain_nginx_port = 8081
configuration.nginx_port = 9443
configuration.server-admin = admin@example.com
configuration.apache_custom_https = ""
configuration.apache_custom_http = ""
configuration.apache-key =
configuration.apache-certificate =
configuration.apache-ca-certificate =
configuration.open-port = 80 443
configuration.extra_slave_instance_list =
configuration.disk-cache-size = 8G
configuration.ram-cache-size = 1G
configuration.trafficserver-autoconf-port = 8083
configuration.trafficserver-mgmt-port = 8084
configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
[frontend-configuration]
template-log-access = ${template-log-access:target}
log-access-configuration = $${directory:etc}/apache-log-access.conf
apache-directory = ${apache:location}
apache-ipv6 = $${instance-parameter:ipv6-random}
apache-https-port = $${instance-parameter:configuration.port}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type instance-parameter:slap-software-type
key slapparameter_dict instance-parameter:configuration
section directory directory
$${:extra-context}
[dynamic-template-default-vh]
< = jinja2-template-base
template = ${template-default-virtualhost:target}
rendered = $${apache-directory:slave-configuration}/000.conf
extensions = jinja2.ext.do
extra-context =
key http_port instance-parameter:configuration.plain_http_port
key https_port instance-parameter:configuration.port
[software-release-path]
template-empty = ${template-empty:target}
template-slave-configuration = ${template-slave-configuration:target}
template-default-slave-virtualhost = ${template-default-slave-virtualhost:target}
template-cached-slave-virtualhost = ${template-cached-slave-virtualhost:target}
template-nginx-eventsource-slave-virtualhost = ${template-nginx-eventsource-slave-virtualhost:target}
template-nginx-notebook-slave-virtualhost = ${template-nginx-notebook-slave-virtualhost:target}
apache-location = ${apache:location}
[dynamic-custom-personal-template-slave-list]
< = jinja2-template-base
template = ${template-slave-list:target}
filename = custom-personal-instance-slave-list.cfg
extensions = jinja2.ext.do
extra-context =
key apache_configuration_directory apache-directory:slave-configuration
key nginx_configuration_directory apache-directory:nginx-slave-configuration
key apache_cached_configuration_directory apache-directory:slave-with-cache-configuration
key slave_with_cache_configuration_directory apache-directory:slave-with-cache-configuration
key http_port instance-parameter:configuration.plain_http_port
key https_port instance-parameter:configuration.port
key nginx_http_port instance-parameter:configuration.plain_nginx_port
key nginx_https_port instance-parameter:configuration.nginx_port
key public_ipv4 instance-parameter:configuration.public-ipv4
key slave_instance_list instance-parameter:slave-instance-list
key extra_slave_instance_list instance-parameter:configuration.extra_slave_instance_list
key custom_ssl_directory apache-directory:vh-ssl
key apache_log_directory apache-directory:slave-log
key local_ipv4 instance-parameter:ipv4-random
key global_ipv6 slap-network-information:global-ipv6
key varnginx directory:varnginx
key empty_template software-release-path:template-empty
key template_custom_slave_configuration software-release-path:template-slave-configuration
key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
key template_cached_slave_configuration software-release-path:template-cached-slave-virtualhost
key template_eventsource_slave_configuration software-release-path:template-nginx-eventsource-slave-virtualhost
key template_notebook_slave_configuration software-release-path:template-nginx-notebook-slave-virtualhost
raw software_type single-custom-personal
key frontend_lazy_graceful_reload frontend-apache-lazy-graceful:rendered
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
section nginx_configuration nginx-configuration
key monitor_base_url monitor-instance-parameter:monitor-base-url
key promise_directory directory:promise
key report_directory directory:bin
raw bin_directory ${buildout:bin-directory}
[dynamic-virtualhost-template-slave]
<= jinja2-template-base
template = ${template-slave-configuration:target}
rendered = $${directory:template}/slave-virtualhost.conf.in
extensions = jinja2.ext.do
extra-context =
key https_port instance-parameter:configuration.port
key http_port instance-parameter:configuration.plain_http_port
key apache_custom_https instance-parameter:configuration.apache_custom_https
key apache_custom_http instance-parameter:configuration.apache_custom_http
# Deploy Apache Frontend (new way, no recipe, jinja power)
[dynamic-apache-frontend-template]
< = jinja2-template-base
template = ${template-apache-frontend-configuration:target}
rendered = $${apache-configuration:frontend-configuration}
extra-context =
key httpd_home software-release-path:apache-location
key httpd_mod_ssl_cache_directory apache-directory:mod-ssl
key domain instance-parameter:configuration.domain
key document_root apache-directory:document-root
key instance_home buildout:directory
key ipv4_addr instance-parameter:ipv4-random
key ipv6_addr instance-parameter:ipv6-random
key http_port instance-parameter:configuration.plain_http_port
key https_port instance-parameter:configuration.port
key server_admin instance-parameter:configuration.server-admin
key protected_path apache-configuration:protected-path
key access_control_string apache-configuration:access-control-string
key login_certificate ca-frontend:cert-file
key login_key ca-frontend:key-file
key login_ca_crt ca-custom-frontend:rendered
key ca_dir certificate-authority:ca-dir
key ca_crl certificate-authority:ca-crl
key access_log apache-configuration:access-log
key error_log apache-configuration:error-log
key pid_file apache-configuration:pid-file
key slave_configuration_directory apache-directory:slave-configuration
key cached_port apache-configuration:cache-through-port
key ssl_cached_port apache-configuration:ssl-cache-through-port
key slave_with_cache_configuration_directory apache-directory:slave-with-cache-configuration
section frontend_configuration frontend-configuration
[apache-frontend]
recipe = slapos.cookbook:wrapper
command-line = ${apache:location}/bin/httpd -f $${dynamic-apache-frontend-template:rendered} -DFOREGROUND
wrapper-path = $${directory:bin}/frontend_apache
[not-found-html]
recipe = slapos.cookbook:symbolic.link
target-directory = $${apache-directory:document-root}
link-binary =
${template-not-found-html:target}
[apache-directory]
recipe = slapos.cookbook:mkdirectory
document-root = $${directory:srv}/htdocs
slave-configuration = $${directory:etc}/apache-slave-conf.d/
slave-with-cache-configuration = $${directory:etc}/apache-slave-with-cache-conf.d/
cache = $${directory:var}/cache
mod-ssl = $${:cache}/httpd_mod_ssl
vh-ssl = $${:slave-configuration}/ssl
slave-log = $${directory:log}/httpd
nginx-slave-configuration = $${directory:etc}/nginx-slave-conf.d/
[apache-configuration]
frontend-configuration = $${directory:etc}/apache_frontend.conf
access-log = $${directory:log}/frontend-apache-access.log
error-log = $${directory:log}/frontend-apache-error.log
pid-file = $${directory:run}/httpd.pid
protected-path = /
access-control-string = none
frontend-configuration-verification = ${apache:location}/bin/httpd -Sf $${:frontend-configuration} > /dev/null
frontend-graceful-command = $${:frontend-configuration-verification}; if [ $? -eq 0 ]; then kill -USR1 $(cat $${:pid-file}); fi
# Comunication with ats
cache-port = $${trafficserver-variable:input-port}
cache-through-port = 26011
ssl-cache-through-port = 26012
# Create wrapper for "apachectl conftest" in bin
[configtest]
recipe = slapos.cookbook:wrapper
command-line = ${apache:location}/bin/httpd -f $${directory:etc}/apache_frontend.conf -t
wrapper-path = $${directory:bin}/apache-configtest
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${directory:service}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
[ca-frontend]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/apache_frontend.key
cert-file = $${cadirectory:certs}/apache_frontend.crt
executable = $${apache-frontend:wrapper-path}
wrapper = $${directory:service}/frontend_apache
key-content = $${instance-parameter:configuration.apache-key}
cert-content = $${instance-parameter:configuration.apache-certificate}
# Put domain name
name = $${instance-parameter:configuration.domain}
[ca-custom-frontend]
< = jinja2-template-base
template = ${template-empty:target}
rendered = $${cadirectory:certs}/apache_frontend.ca.crt
extra-context =
key content instance-parameter:configuration.apache-ca-certificate
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${directory:service}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${directory:bin}/cron_simplelogger
log = $${directory:log}/cron.log
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
# Deploy Logrotate
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${directory:bin}/logrotate
conf = $${directory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${directory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-configuration:error-log} $${apache-configuration:access-log}
frequency = daily
rotatep-num = 30
post = $${frontend-apache-lazy-graceful:rendered} &
sharedscripts = true
notifempty = true
create = true
[logrotate-entry-nginx]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache-nginx
log = $${nginx-configuration:error_log} $${nginx-configuration:access_log}
frequency = daily
rotatep-num = 30
post = $${nginx-configuration:nginx-graceful-command}
sharedscripts = true
notifempty = true
create = true
#################
# Trafficserver
#################
[trafficserver-directory]
recipe = slapos.cookbook:mkdirectory
configuration = $${directory:etc}/trafficserver
local-state = $${directory:var}/trafficserver
bin_path = ${trafficserver:location}/bin
log = $${directory:log}/trafficserver
cache-path = $${directory:srv}/ats_cache
[trafficserver-variable]
wrapper-path = $${directory:service}/trafficserver
reload-path = $${directory:etc-run}/trafficserver-reload
local-ip = $${instance-parameter:ipv4-random}
input-port = 23432
hostname = $${instance-parameter:configuration.frontend-name}
remap = map /HTTPS/ http://$${instance-parameter:ipv4-random}:$${apache-configuration:ssl-cache-through-port}
map / http://$${instance-parameter:ipv4-random}:$${apache-configuration:cache-through-port}
plugin-config = ${trafficserver:location}/libexec/trafficserver/rfc5861.so
cache-path = $${trafficserver-directory:cache-path}
disk-cache-size = $${instance-parameter:configuration.disk-cache-size}
autoconf-port = $${instance-parameter:configuration.trafficserver-autoconf-port}
mgmt-port = $${instance-parameter:configuration.trafficserver-mgmt-port}
ram-cache-size = $${instance-parameter:configuration.ram-cache-size}
[trafficserver-configuration-directory]
recipe = plone.recipe.command
command = cp -rn ${trafficserver:location}/etc/trafficserver/* $${:target}
target = $${trafficserver-directory:configuration}
[trafficserver-launcher]
recipe = slapos.cookbook:wrapper
command-line = ${trafficserver:location}/bin/traffic_cop
wrapper-path = $${trafficserver-variable:wrapper-path}
environment = TS_ROOT=$${buildout:directory}
[trafficserver-reload]
recipe = slapos.cookbook:wrapper
command-line = ${trafficserver:location}/bin/traffic_line -x
wrapper-path = $${trafficserver-variable:reload-path}
environment = TS_ROOT=$${buildout:directory}
# XXX Dedicated Jinja Section without slapparameter
[trafficserver-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${trafficserver-directory:configuration}/$${:filename}
extra-context =
mode = 600
context =
section ats_directory trafficserver-directory
section ats_configuration trafficserver-variable
$${:extra-context}
[trafficserver-records-config]
< = trafficserver-jinja2-template-base
template = ${template-trafficserver-records-config:location}/${template-trafficserver-records-config:filename}
filename = records.config
extra-context =
import os_module os
[trafficserver-storage-config]
< = trafficserver-jinja2-template-base
template = ${template-trafficserver-storage-config:location}/${template-trafficserver-storage-config:filename}
filename = storage.config
[trafficserver-remap-config]
< = trafficserver-jinja2-template-base
template = ${template-empty:target}
filename = remap.config
context =
key content trafficserver-variable:remap
[trafficserver-plugin-config]
< = trafficserver-jinja2-template-base
template = ${template-empty:target}
filename = plugin.config
context =
key content trafficserver-variable:plugin-config
[trafficserver-promise-listen-port]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/trafficserver-port-listening
hostname = $${trafficserver-variable:local-ip}
port = $${trafficserver-variable:input-port}
[trafficserver-line]
recipe = slapos.cookbook:wrapper
command-line = ${trafficserver:location}/bin/traffic_line
wrapper-path = $${directory:bin}/traffic_line
environment = TS_ROOT=$${buildout:directory}
[trafficserver-promise-cache-availability]
recipe = collective.recipe.template
input =
inline:#!${buildout:executable}
import subprocess
import sys
traffic_line = "$${trafficserver-line:wrapper-path}"
result = float(subprocess.check_output([traffic_line, '-r', 'proxy.node.cache.percent_free' ]))
if result != 0: sys.exit(0)
sys.stderr.write("Cache not available, availability: %s" % result)
sys.exit(127)
output = $${directory:promise}/trafficserver-cache-availability
mode = 700
### End of ATS sections
### Apaches Graceful and promises
[frontend-apache-graceful-bin]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:bin}/frontend-apache-safe-graceful
mode = 0700
extra-context =
key content apache-configuration:frontend-graceful-command
[frontend-apache-graceful]
< = jinja2-template-base
template = ${template-apache-graceful-script:target}
rendered = $${directory:etc-run}/frontend-apache-safe-graceful
mode = 0700
extra-context =
key directory_run directory:run
key directory_etc directory:etc
key apache_graceful_reload_command apache-configuration:frontend-graceful-command
[frontend-apache-lazy-graceful]
< = jinja2-template-base
template = ${template-apache-lazy-script-call:target}
rendered = $${directory:bin}/frontend-apache-lazy-graceful
mode = 0700
pid-file = $${directory:run}/lazy-graceful.pid
extra-context =
key pid_file :pid-file
raw wait_time 60
key lazy_command apache-configuration:frontend-graceful-command
# Promises checking configuration:
[promise-frontend-apache-configuration]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:promise}/frontend-apache-configuration-promise
mode = 0700
extra-context =
key content apache-configuration:frontend-configuration-verification
[promise-apache-frontend-v4-https]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_frontend_ipv4_https
hostname = $${instance-parameter:ipv4-random}
port = $${instance-parameter:configuration.port}
[promise-apache-frontend-v4-http]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_frontend_ipv4_http
hostname = $${instance-parameter:ipv4-random}
port = $${instance-parameter:configuration.plain_http_port}
[promise-apache-frontend-v6-https]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_frontend_ipv6_https
hostname = $${instance-parameter:ipv6-random}
port = $${instance-parameter:configuration.port}
[promise-apache-frontend-v6-http]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_frontend_ipv6_http
hostname = $${instance-parameter:ipv6-random}
port = $${instance-parameter:configuration.plain_http_port}
[promise-apache-frontend-cached]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_cached
hostname = $${instance-parameter:ipv4-random}
port = $${apache-configuration:cache-through-port}
[promise-apache-frontend-ssl-cached]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/apache_ssl_cached
hostname = $${instance-parameter:ipv4-random}
port = $${apache-configuration:ssl-cache-through-port}
[promise-apache-is-process-older-than-dependency-set]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/is-process-older-than-dependency-set $${apache-configuration:pid-file}
wrapper-path = $${directory:promise}/apache-frontend-is-running-actual-software-release
[promise-apache-mpm-watchdog]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/apache-mpm-watchdog --url http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u admin -p $${monitor-htpasswd:passwd} -d $${directory:run}/apache-mpm-watchdog.json
wrapper-path = $${directory:promise}/apache-mpm-watchdog
[slap_connection]
# Kept for backward compatiblity
computer_id = $${slap-connection:computer-id}
partition_id = $${slap-connection:partition-id}
server_url = $${slap-connection:server-url}
software_release_url = $${slap-connection:software-release-url}
key_file = $${slap-connection:key-file}
cert_file = $${slap-connection:cert-file}
[slap-parameter]
# Define default parameter(s) that will be used later, in case user didn't
# specify it
# All parameters are available through the configuration.XX syntax.
# All possible parameters should have a default.
domain = example.org
public-ipv4 =
port = 4443
plain_http_port = 8080
server-admin = admin@example.com
apache_custom_https = ""
apache_custom_http = ""
apache-key =
apache-certificate =
open-port = 80 443
extra_slave_instance_list =
frontend-name =
monitor-cors-domains =
monitor-httpd-port = 8072
monitor-username = $${monitor-instance-parameter:username}
monitor-password = $${monitor-htpasswd:passwd}
#######
# Monitoring sections
#
[monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
cors-domains = $${slap-parameter:monitor-cors-domains}
username = $${slap-parameter:monitor-username}
password = $${slap-parameter:monitor-password}
[monitor-conf-parameters]
private-path-list +=
$${directory:logrotate-backup}
[monitor-traffic-summary-last-stats-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:bin}/traffic-summary-last-stats_every_1_hour
mode = 0700
command = export TS_ROOT=$${buildout:directory} && echo "$(${trafficserver:location}/bin/traffic_logstats -f $${trafficserver-directory:log}/squid.blog)
"
extra-context =
key content monitor-traffic-summary-last-stats-wrapper:command
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:bin}/ats-cache-stats_every_1_hour
mode = 0700
command = export TS_ROOT=$${buildout:directory} && echo "$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})
"
extra-context =
key content monitor-ats-cache-stats-wrapper:command
[monitor-apache-server-status-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:bin}/monitor-apache-server-status-wrapper
mode = 0700
command = ${curl:location}/bin/curl -s http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u $${monitor-instance-parameter:username}:$${monitor-htpasswd:passwd} 2>&1
extra-context =
key content monitor-apache-server-status-wrapper:command
[monitor-ats-cache-stats-config]
< = jinja2-template-base
template = ${template-empty:target}
rendered = $${trafficserver-configuration-directory:target}/cache-config.stats
mode = 644
context =
raw content show:cache-stats
[monitor-verify-re6st-connectivity]
recipe = slapos.cookbook:check_url_available
path = $${directory:promise}/re6st-connectivity
url = $${instance-parameter:configuration.re6st-verification-url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
#######################
# Nginx
#
[nginx-frontend]
recipe = slapos.cookbook:wrapper
command-line = ${nginx-push-stream:location}/sbin/nginx -c $${dynamic-nginx-frontend-template:rendered}
wrapper-path = $${directory:service}/frontend_nginx
[dynamic-nginx-frontend-template]
< = jinja2-template-base
template = ${template-nginx-configuration:output}
rendered = $${directory:etc}/nginx.cfg
mode = 0600
extra-context =
key ip nginx-configuration:ip
key local_ip nginx-configuration:local_ip
key port nginx-configuration:port
key plain_port nginx-configuration:plain_port
key worker_processes nginx-configuration:worker_processes
key pidfile nginx-configuration:pid-file
key worker_connections nginx-configuration:worker_connections
key error_log nginx-configuration:error_log
key access_log nginx-configuration:access_log
key ssl_certificate ca-frontend:cert-file
key ssl_key ca-frontend:key-file
key varnginx directory:varnginx
key slave_configuration_directory nginx-configuration:slave-configuration-directory
key document_root apache-directory:document-root
[nginx-configuration]
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
local_ip = $${slap-network-information:local-ipv4}
port = $${instance-parameter:configuration.nginx_port}
plain_port = $${instance-parameter:configuration.plain_nginx_port}
worker_processes = 4
worker_connections = 1024
slave-configuration-directory = $${apache-directory:nginx-slave-configuration}
pid-file = $${directory:run}/nginx.pid
nginx-graceful-command = $${:nginx-configuration-verification}; if [ $? -eq 0 ]; then kill -HUP $(cat $${:pid-file}); fi
nginx-configuration-verification = ${nginx-push-stream:location}/sbin/nginx -t -c $${dynamic-nginx-frontend-template:rendered}
[frontend-nginx-graceful]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:etc-run}/frontend-nginx-safe-graceful
mode = 0700
extra-context =
key content nginx-configuration:nginx-graceful-command
[promise-nginx-configuration]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:promise}/nginx-configuration-promise
mode = 0700
extra-context =
key content nginx-configuration:nginx-configuration-verification
[promise-nginx-frontend-v4-https]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/nginx_frontend_ipv4_https
hostname = $${instance-parameter:ipv4-random}
port = $${instance-parameter:configuration.nginx_port}
[promise-nginx-frontend-v4-http]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/nginx_frontend_ipv4_http
hostname = $${instance-parameter:ipv4-random}
port = $${instance-parameter:configuration.plain_nginx_port}
[promise-nginx-frontend-v6-https]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/nginx_frontend_ipv6_https
hostname = $${instance-parameter:ipv6-random}
port = $${instance-parameter:configuration.nginx_port}
[promise-nginx-frontend-v6-http]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/nginx_frontend_ipv6_http
hostname = $${instance-parameter:ipv6-random}
port = $${instance-parameter:configuration.plain_nginx_port}
[promise-nginx-is-process-older-than-dependency-set]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/is-process-older-than-dependency-set $${nginx-configuration:pid-file}
wrapper-path = $${directory:promise}/promise-nginx-is-process-older-than-dependency-set
[port-redirection]
<= jinja2-template-base
template = inline:
[{"srcPort": 80, "destPort": {{ http_port }}}, {"srcPort": 443, "destPort": {{ https_port }}}]
rendered = $${buildout:directory}/.slapos-port-redirect
mode = 0644
extra-context =
key http_port instance-parameter:configuration.plain_http_port
key https_port instance-parameter:configuration.port
instance-apache-input-schema.json 0000664 0000000 0000000 00000006326 13712426147 0034512 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"public-ipv4": {
"title": "Public IPv4",
"description": "Public ipv4 of the frontend (the one Apache will be indirectly listening to).",
"type": "string"
},
"ip-read-limit": {
"title": "IPReadLimit",
"description": "Value used to set IPReadLimit Parameter for antiloris.",
"type": "integer",
"default": 10
},
"mpm-server-limit": {
"title": "ServerLimit",
"description": "Value used to set ServerLimit on apache configuration.",
"type": "integer",
"default": 16
},
"mpm-max-clients": {
"title": "MaxClients",
"description": "Value used to set MaxClients on apache configuration.",
"type": "integer",
"default": 400
},
"mpm-start-servers": {
"title": "StartServers",
"description": "Value used to set StartServers on apache configuration.",
"type": "integer",
"default": 3
},
"mpm-thread-per-child": {
"title": "ThreadsPerChild",
"description": "Value used to set ThreadsPerChild on apache configuration.",
"type": "integer",
"default": 25
},
"mpm-graceful-shutdown-timeout": {
"title": "ThreadsPerChild",
"description": "Value used to set ThreadsPerChild on apache configuration.",
"type": "integer",
"default": 5
},
"enable-http2-by-default": {
"title": "Enable HTTP2 by Default",
"description": "Use HTTP2 as default Protocol",
"type": "string",
"default": "true",
"enum": [
"true",
"false"
]
},
"re6st-verification-url": {
"title": "Test Verification URL",
"description": "Url to verify if the internet and/or re6stnet is working.",
"type": "string"
},
"-frontend-authorized-slave-string": {
"title": "Authorized Slave String",
"description": "List of SOFTINST-XXX of Slaves, separated by space which is allowed to use custom configuration.",
"type": "string"
},
"apache-key": {
"title": "Apache Key",
"description": "Apache Key",
"textarea": true,
"type": "string"
},
"apache-certificate": {
"title": "Apache Certificate",
"description": "Apache Certificate",
"textarea": true,
"type": "string"
},
"apache-ca-certificate": {
"title": "Apache CA Certificate",
"description": "Apache CA Certificate",
"textarea": true,
"type": "string"
},
"domain": {
"title": "Domain",
"description": "Base Domain for create subdomains (ie.: example.com).",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"nginx-domain": {
"title": "Nginx Domain",
"description": "Base Domain for create subdomains (ie.: example2.com) for websocket, notebook and eventsource.",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"-frontend-quantity": {
"title": "Frontend Replication Quantity",
"description": "Quantity of Frontends Replicate.",
"type": "integer"
}
}
}
instance-apache-replicate.cfg.in 0000664 0000000 0000000 00000016071 13712426147 0034256 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend {% if slap_software_type in software_type -%}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type slap-parameter:slap_software_type
key slave_instance_list slap-parameter:slave_instance_list
${:extra-context}
{% set part_list = [] -%}
{% set single_type_key = 'single-' %}
{% if slap_software_type == "replicate" %}
{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') -%}
{% elif slap_software_type in ['default', 'RootSoftwareInstance'] -%}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') -%}
{% else -%}
{% set frontend_type = "%s%s" % (single_type_key, slap_software_type) -%}
{% endif -%}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int -%}
{% set slave_list_name = 'extra_slave_instance_list' -%}
{% set frontend_list = [] %}
{% set frontend_section_list = [] %}
{% set request_dict = {} %}
{% set namebase = 'apache-frontend' -%}
# XXX Dirty hack, not possible to define default value before
{% set sla_computer_apache_1_key = '-sla-1-computer_guid' -%}
{% if not sla_computer_apache_1_key in slapparameter_dict -%}
{% do slapparameter_dict.__setitem__(sla_computer_apache_1_key, '${slap-connection:computer-id}') -%}
{% endif -%}
# Here we request individualy each frontend.
# The presence of sla parameters is checked and added if found
{% for i in range(1, frontend_quantity + 1) -%}
{% set frontend_name = "%s-%s" % (namebase, i) -%}
{% set request_section_title = 'request-%s' % frontend_name -%}
{% set sla_key = "-sla-%s-" % i -%}
{% set sla_key_length = sla_key | length %}
{% set sla_dict = {} %}
{% set config_key = "-frontend-config-%s-" % i %}
{% set config_key_length = config_key | length %}
{% set config_dict = {} %}
{% for key in slapparameter_dict.keys() %}
{% if key.startswith(sla_key) %}
{% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
# We check for specific configuration regarding the frontend
{% elif key.startswith(config_key) -%}
{% do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %}
{% endif -%}
{% endfor -%}
{% do frontend_list.append(frontend_name) -%}
{% do frontend_section_list.append(request_section_title) -%}
{% do part_list.append(request_section_title) -%}
# Filling request dict for slave
{% set state_key = "-frontend-%s-state" % i %}
{% do request_dict.__setitem__(request_section_title,
{
'config': config_dict,
'name': frontend_name,
'sla': sla_dict,
'state': slapparameter_dict.pop(state_key, None)
}) %}
{% endfor -%}
{% set authorized_slave_string = slapparameter_dict.pop('-frontend-authorized-slave-string', '') -%}
{% set authorized_slave_list = [] %}
{% set rejected_slave_list = [] %}
{% for slave in slave_instance_list %}
{% if not (slave.has_key('apache_custom_http') and not slave.get('slave_reference') in authorized_slave_string) %}
{% do authorized_slave_list.append(slave) %}
{% else %}
{% do rejected_slave_list.append(slave.get('slave_reference')) %}
{% endif %}
{% endfor -%}
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
{% else %}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
{% for section, frontend_request in request_dict.iteritems() %}
[{{section}}]
<= replicate
name = {{ frontend_request.get('name') }}
{% if frontend_request.get('state') %}
state = {{ frontend_request.get('state') }}
{% endif%}
{% set slave_configuration_dict = frontend_request.get('config') %}
{% do slave_configuration_dict.update(**slapparameter_dict) %}
{% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
config-_ = {{ json_module.dumps(slave_configuration_dict) }}
{% if frontend_request.get('sla') %}
{% for parameter, value in frontend_request.get('sla').iteritems() -%}
sla-{{ parameter }} = {{ value }}
{% endfor -%}
{% endif -%}
{% endfor -%}
[publish-information]
<= monitor-publish
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_list | length }}
rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
#----------------------------
#--
#-- Publish slave information
[publish-slave-information]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-publish-slave-information:rendered}
RootSoftwareInstance = ${dynamic-publish-slave-information:rendered}
replicate = ${dynamic-publish-slave-information:rendered}
custom-personal = ${dynamic-publish-slave-information:rendered}
custom-group = ${dynamic-publish-slave-information:rendered}
[slave-information]
{% for frontend_section in frontend_section_list -%}
{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
{% endfor -%}
[dynamic-publish-slave-information]
< = jinja2-template-base
template = {{ template_publish_slave_information }}
filename = dynamic-publish-slave-information.cfg
extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
[monitor-conf-parameters]
monitor-url-list +=
{% for frontend in frontend_section_list %}
{{ ' ${' + frontend + ':connection-monitor-base-url}' }}
{% endfor -%}
[buildout]
extends = {{ template_monitor }}
parts =
monitor-base
publish-slave-information
publish-information
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
# publish-information
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap_connection]
# Kept for backward compatiblity
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
[slap-parameter]
slave_instance_list =
-frontend-quantity = 1
-frontend-type = single-default
{%- endif %}
instance-output-schema.json 0000664 0000000 0000000 00000002375 13712426147 0033474 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Apache Frontend instanciation",
"properties": {
"accepted-slave-amount": {
"description": "Amount of Slaves allocated to the Instance which are deployed",
"type": "integer"
},
"slave-amount": {
"description": "Total amount of Slaves allocated to the Instance (include blocked ones)",
"type": "integer"
},
"reject-slave-amount": {
"description": "Rejected Amount of Slaves allocated to the Instance which are not deployed",
"type": "integer"
},
"rejected-slave-list": {
"description": "List of slave instances references which are rejected",
"type": "array"
},
"domain": {
"description": "Base domain used by the instance",
"type": "string"
},
"monitor-base-url": {
"description": "Base url for monitor",
"type": "string"
},
"monitor-url": {
"description": "Url of Monitor (opml)",
"type": "string"
},
"monitor-password": {
"description": "Password to access the monitor",
"type": "string"
},
"monitor-user": {
"description": "User to access the monitor",
"type": "string"
}
},
"type": "object"
}
instance-slave-apache-input-schema.json 0000664 0000000 0000000 00000015070 13712426147 0035616 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"url": {
"title": "Backend URL",
"description": "Url of the backend",
"type": "string",
"pattern": "^(http|https|ftp)://"
},
"https-url": {
"title": "HTTPS Backend URL",
"description": "Url of the backend if it is diferent from url parameter",
"type": "string",
"pattern": "^(http|https|ftp)://"
},
"custom_domain": {
"title": "Custom Domain",
"description": "Custom Domain to use for the website",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"server-alias": {
"title": "Server Alias",
"description": "Server Alias List separated by space",
"type": "string",
"default": ""
},
"type": {
"title": "Backend Type",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster",
"type": "string",
"default": "",
"enum": [
"",
"zope",
"redirect",
"notebook",
"eventsource"
]
},
"path": {
"title": "Backend Path",
"description": "Path to proxy to in the backend",
"type": "string",
"default": ""
},
"default-path": {
"title": "Default Path",
"description": "Provide default path to redirect user to when user access / (the site root)",
"type": "string",
"default": ""
},
"ssl_crt": {
"title": "SSL Certificate",
"description": "Content of the SSL Certificate file",
"type": "string",
"textarea": true,
"default": ""
},
"ssl_key": {
"title": "SSL Key",
"description": "Content of the SSL Key file",
"type": "string",
"textarea": true,
"default": ""
},
"ssl_ca_crt": {
"title": "SSL Certificate Authority's Certificate",
"description": "Content of the CA certificate file",
"type": "string",
"textarea": true,
"default": ""
},
"https-only": {
"title": "HTTPS Only",
"description": "If set to true, http requests will be redirected to https",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"ssl-proxy-verify": {
"title": "Verify Backend Certificates",
"description": "If set to true, Backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"ssl_proxy_ca_crt": {
"title": "SSL Backend Authority's Certificate",
"description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
"type": "string",
"textarea": true,
"default": ""
},
"monitor-ipv6-test": {
"title": "IPv6 Address to Monitor Packet Lost",
"description": "IPv6 Address for the frontend keep monitoring with ping6 (without brackets)",
"type": "string",
"default": ""
},
"monitor-ipv4-test": {
"title": "IPv4 Address to Monitor Packet Lost",
"description": "IPv4 Address for the frontend keep monitoring with ping",
"type": "string",
"default": ""
},
"re6st-optimal-test": {
"title": "IPv6 and IPv4 Address to test Re6st",
"description": "IPv6 and IPv6 Address for the frontend test if re6st is on the optimal status (use ipv6,ipv4)",
"type": "string",
"default": ""
},
"enable_cache": {
"title": "Enable Cache",
"description": "If set to true, http caching server (Apache Traffic Server) will be used between frontend apache and backend",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"disable-no-cache-request": {
"title": "Disable 'no-cache' requests",
"description": "If set to true, Cache-Control and Pragma requests headers will not be sent to cache and backend servers. This prevents clients from bypassing cache when enable_cache is true",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"disable-via-header": {
"title": "Disable 'Via' headers from cache",
"description": "If set to true, Via response headers will not be sent to client",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"enable-http2": {
"title": "Enable HTTP2 Protocol",
"description": "Use HTTP2 Protocol for the site",
"type": "string",
"default": "true",
"enum": [
"true",
"false"
]
},
"prefer-gzip-encoding-to-backend": {
"title": "Prefer gzip Encoding for Backend",
"description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
"type": "string",
"default": "false",
"enum": [
"false",
"true"
]
},
"disabled-cookie-list": {
"title": "Disabled Cookies",
"description": "List of Cookies separated by space that will not be sent to cache and backend servers. This is especially useful to discard analytics tracking cookies when using Vary: Cookie cache headers",
"type": "string",
"default": ""
},
"apache_custom_http": {
"title": "HTTP configuration",
"description": "Raw http configuration in python template format. Your site will be rejected if you use it without notification and approval of frontend administrators",
"default": "",
"textarea": true,
"type": "string"
},
"apache_custom_https": {
"title": "HTTPS configuration",
"description": "Raw https configuration in python template format. Your site will be rejected if you use it without notification and approval of frontend administrators",
"default": "",
"textarea": true,
"type": "string"
},
"virtualhostroot-http-port": {
"description": "Port where http requests to frontend will be redirected.",
"default": 80,
"type": "integer"
},
"virtualhostroot-https-port": {
"description": "Port where https requests to frontend will be redirected.",
"default": 443,
"type": "integer"
}
}
}
instance-slave-output-schema.json 0000664 0000000 0000000 00000001567 13712426147 0034606 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Apache Frontend instanciation",
"properties": {
"replication_number": {
"description": "Number of nodes the slave is replicated",
"type": "integer"
},
"domain": {
"description": "Base domain used by the instance",
"type": "string"
},
"url": {
"description": "Default URL provided",
"type": "string"
},
"site_url": {
"description": "URL for HTTP access",
"type": "string"
},
"secure_access": {
"description": "URL for HTTP access",
"type": "string"
},
"public-ipv4": {
"description": "Public IPv4 to be included on DNS",
"type": "string"
},
"log-access-url": {
"description": "List of URLs to access logs",
"type": "array"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/instance.cfg 0000664 0000000 0000000 00000003502 13712426147 0030536 0 ustar 00root root 0000000 0000000 [buildout]
parts =
dynamic-template-apache-replicate
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type slap-parameters:slap-software-type
key slapparameter_dict slap-parameters:configuration
key slave_instance_list slap-parameters:slave-instance-list
$${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${dynamic-template-apache-replicate:rendered}
RootSoftwareInstance = $${dynamic-template-apache-replicate:rendered}
custom-personal = $${dynamic-template-apache-replicate:rendered}
single-default = ${template-apache-frontend:output}
single-custom-personal = ${template-apache-frontend:output}
replicate = $${dynamic-template-apache-replicate:rendered}
[dynamic-template-apache-replicate]
< = jinja2-template-base
template = ${template-apache-replicate:target}
filename = instance-apache-replicate.cfg
extensions = jinja2.ext.do
extra-context =
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor ${monitor2-template:rendered}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/software.cfg 0000664 0000000 0000000 00000000422 13712426147 0030562 0 ustar 00root root 0000000 0000000 [buildout]
extends = common.cfg
[versions]
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
ecdsa = 0.13
gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
websockify = 0.8.0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/software.cfg.json 0000664 0000000 0000000 00000002202 13712426147 0031530 0 ustar 00root root 0000000 0000000 {
"name": "Apache Frontend",
"description": "Apache Frontend",
"serialisation": "xml",
"software-type": {
"custom-personal-slave": {
"title": "Custom Personal (Slave)",
"description": "Custom Personal",
"request": "instance-slave-apache-input-schema.json",
"response": "instance-output-schema.json",
"index": 1,
"software-type": "custom-personal",
"shared": true
},
"custom-personal": {
"title": "Custom Personal",
"description": "",
"request": "instance-apache-input-schema.json",
"response": "instance-output-schema.json",
"index": 2
},
"default": {
"title": "Default",
"software-type": "default",
"description": "Default",
"request": "instance-apache-input-schema.json",
"response": "instance-output-schema.json",
"index": 3
},
"default-slave": {
"title": "Default (slave)",
"software-type": "default",
"description": "Default",
"request": "instance-slave-apache-input-schema.json",
"response": "instance-output-schema.json",
"index": 4,
"shared": true
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates/ 0000775 0000000 0000000 00000000000 13712426147 0030247 5 ustar 00root root 0000000 0000000 000.conf.in 0000664 0000000 0000000 00000001035 13712426147 0031742 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates
ServerName www.example.org
SSLEngine on
SSLProxyEngine on
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
# Rewrite part
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
ErrorDocument 404 /notfound.html
ServerName www.example.org
ErrorDocument 404 /notfound.html
apache-custom-slave-list.cfg.in 0000664 0000000 0000000 00000035127 13712426147 0036100 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% if software_type == slap_software_type -%}
{% set cached_server_dict = {} -%}
{% set part_list = [] -%}
{% set cache_port = apache_configuration.get('cache-port') %}
{% set cached_port = apache_configuration.get('cache-through-port') %}
{% set ssl_cached_port = apache_configuration.get('ssl-cache-through-port') %}
{% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
{% set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) -%}
{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{% set NGINX_TYPE_LIST = ['eventsource', 'notebook'] -%}
{% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
{% set slave_log_dict = {} -%}
{% if extra_slave_instance_list -%}
{% set slave_instance_information_list = [] -%}
{% set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
{% endif -%}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
${:extra-context}
{% do logrotate_dict.pop('recipe') %}
[logrotate]
{% for key, value in logrotate_dict.iteritems() -%}
{{ key }} = {{ value }}
{% endfor %}
post = {{ frontend_lazy_graceful_reload }} &
frequency = daily
rotatep-num = 30
sharedscripts = true
notifempty = true
create = true
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = {{ custom_ssl_directory }}/requests/
private = {{ custom_ssl_directory }}/private/
certs = {{ custom_ssl_directory }}/certs/
newcerts = {{ custom_ssl_directory }}/newcerts/
crl = {{ custom_ssl_directory }}/crl/
{# Loop throught slave list to set up slaves #}
{% for slave_instance in slave_instance_list -%}
{% set slave_reference = slave_instance.get('slave_reference') -%}
{% set slave_type = slave_instance.get('type', '') -%}
{% set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference -%}
{% set slave_parameter_dict = generic_instance_parameter_dict.copy() -%}
{% set slave_publish_dict = {} -%}
{% set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
{% set slave_logrotate_section = slave_reference + "-logs" -%}
{% set slave_password_section = slave_reference + "-password" -%}
{% set slave_ln_section = slave_reference + "-ln" -%}
{% set slave_htaccess_section = slave_reference + '-htaccess' %}
{# extend parts #}
{% do part_list.extend([slave_htaccess_section, slave_ln_section]) -%}
{% do part_list.extend([slave_logrotate_section, slave_section_title]) -%}
{% set slave_log_folder = logrotate_dict.get('backup') + '/' + slave_reference + "-logs" -%}
{# Set Up log files #}
{% do slave_parameter_dict.__setitem__('access_log', '/'.join([apache_log_directory, '%s_access_log' % slave_reference])) -%}
{% do slave_parameter_dict.__setitem__('error_log', '/'.join([apache_log_directory, '%s_error_log' % slave_reference])) -%}
{% do slave_instance.__setitem__('access_log', slave_parameter_dict.get('access_log')) -%}
{% do slave_instance.__setitem__('error_log', slave_parameter_dict.get('error_log')) -%}
{# Add slave log directory to the slave log access dict #}
{% do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
{% set slave_log_access_url = 'https://' + slave_reference + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('apache-ipv6') + ']:' + frontend_configuration.get('apache-https-port') + '/' + slave_reference.lower() + '/' %}
{% do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
{% do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
{% do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
{# Set slave domain if none was defined #}
{% if slave_instance.get('custom_domain', None) == None -%}
{% set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() -%}
{% if slave_type in NGINX_TYPE_LIST -%}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('nginx-domain', slapparameter_dict.get('domain')))) -%}
{% else -%}
{% do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) -%}
{% endif -%}
{% endif -%}
{% set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') -%}
{% if enable_cache and 'url' in slave_instance -%}
{% if 'domain' in slave_instance -%}
{% do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) -%}
{% endif -%}
{% do slave_instance.__setitem__('backend_url', slave_instance.get('url')) -%}
{% do slave_instance.__setitem__('https_backend_url', slave_instance.get('https-url', slave_instance.get('url'))) -%}
{% do slave_instance.__setitem__('url', cache_access) -%}
{% do slave_instance.__setitem__('https-url', ssl_cache_access) -%}
{% do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) -%}
{% endif -%}
{% if not slave_instance.has_key('apache_custom_http') and not slave_instance.has_key('apache_custom_https') %}
{% do slave_publish_dict.__setitem__('domain', slave_instance.get('custom_domain')) -%}
{% do slave_publish_dict.__setitem__('url', "http://%s" % slave_instance.get('custom_domain')) -%}
{% do slave_publish_dict.__setitem__('site_url', "http://%s" % slave_instance.get('custom_domain')) -%}
{% do slave_publish_dict.__setitem__('secure_access', 'https://%s' % slave_instance.get('custom_domain')) -%}
{% endif -%}
[slave-log-directories]
{{slave_reference}} = {{ slave_log_folder }}
{# Set slave logrotate entry #}
[{{slave_logrotate_section}}]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = ${:_buildout_section_name_}
log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
backup = {{ slave_log_folder }}
{# integrate current logs inside #}
[{{slave_ln_section}}]
recipe = plone.recipe.command
stop-on-error = false
command = ln -s {{slave_parameter_dict.get('error_log')}} {{ slave_log_folder }}/apache-error.log && ln -s {{slave_parameter_dict.get('access_log')}} {{ slave_log_folder }}/apache-access.log
{# Set password for slave #}
[{{slave_password_section}}]
recipe = slapos.cookbook:generate.password
storage-path = {{apache_configuration_directory}}/.{{slave_reference}}.passwd
bytes = 8
{# Set up htaccess file for slave #}
[{{slave_htaccess_section}}]
recipe = plone.recipe.command
stop-on-error = true
htaccess-path = {{apache_configuration_directory}}/.{{slave_reference}}.htaccess
command = {{frontend_configuration.get('apache-directory')}}/bin/htpasswd -cb ${:htaccess-path} {{ slave_reference }} {{ '${' + slave_password_section + ':passwd}' }}
{# ################################################## #}
{# Set Slave Certificates if needed #}
{# Set ssl certificates for each slave #}
{% for cert_name in ('ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')-%}
{% if cert_name in slave_instance -%}
{% set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) -%}
{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
{% do part_list.append(cert_title) -%}
{% do slave_parameter_dict.__setitem__(cert_name, cert_file) -%}
{% do slave_instance.__setitem__('path_to_' + cert_name, cert_file) -%}
{# Store certificates on fs #}
[{{ cert_title }}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
extra-context =
key content {{ cert_title + '-config:value' }}
# Store certificate in config
[{{ cert_title + '-config' }}]
value = {{ dumps(slave_instance.get(cert_name)) }}
{% endif -%}
{% endfor -%}
{% if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance -%}
{% set cert_title = '%s-crt' % (slave_reference) -%}
{% set key_title = '%s-key' % (slave_reference) -%}
{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
{% set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) -%}
{% do part_list.append(cert_title) -%}
{% do slave_parameter_dict.__setitem__("ssl_crt", cert_file) -%}
{% do slave_parameter_dict.__setitem__("ssl_key", key_file) -%}
{% do slave_instance.__setitem__('path_to_ssl_crt', cert_file) -%}
{% do slave_instance.__setitem__('path_to_ssl_key', key_file) -%}
[{{cert_title}}]
recipe = slapos.cookbook:certificate_authority.request
#openssl-binary = ${openssl:location}/bin/openssl
requests-directory = ${cadirectory:requests}
ca-private = ${cadirectory:private}
ca-certs = ${cadirectory:certs}
ca-newcerts = ${cadirectory:newcerts}
ca-crl = ${cadirectory:crl}
key-file = {{ key_file }}
cert-file = {{ cert_file }}
key-content = {{ dumps(slave_instance.get('ssl_key')) }}
cert-content = {{ dumps(slave_instance.get('ssl_crt')) }}
{% endif -%}
{# ########################################## #}
{# Set Slave Configuration #}
[{{ slave_configuration_section_name }}]
{% set apache_custom_http = ((slave_instance.pop('apache_custom_http', '')) % slave_parameter_dict) -%}
{% set apache_custom_https = ((slave_instance.pop('apache_custom_https', '')) % slave_parameter_dict) -%}
apache_custom_http = {{ dumps(apache_custom_http) }}
apache_custom_https = {{ dumps(apache_custom_https) }}
{{ '\n' }}
{% for key, value in slave_instance.iteritems() -%}
{{ key }} = {{ dumps(value) }}
{% endfor %}
[{{ slave_section_title }}]
< = jinja2-template-base
{% if slave_type in NGINX_TYPE_LIST %}
rendered = {{ nginx_configuration_directory }}/${:filename}
{% else %}
rendered = {{ apache_configuration_directory }}/${:filename}
{% endif %}
{% if apache_custom_http %}
template = {{ template_custom_slave_configuration }}
{% elif slave_type == 'eventsource' %}
template = {{ template_eventsource_slave_configuration }}
{% elif slave_type == 'notebook' %}
template = {{ template_notebook_slave_configuration }}
{% else %}
template = {{ template_default_slave_configuration }}
{% endif %}
filename = {{ '%s.conf' % slave_reference }}
extra-context =
raw https_port {{ https_port }}
raw http_port {{ http_port }}
raw global_ipv6 {{ global_ipv6 }}
raw local_ipv4 {{ local_ipv4 }}
raw nginx_http_port {{ nginx_http_port }}
raw nginx_https_port {{ nginx_https_port }}
section slave_parameter {{ slave_configuration_section_name }}
{{ '\n' }}
{% set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %}
{% if monitor_ipv6_test %}
{% set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %}
{% do part_list.append(monitor_ipv6_section_title) -%}
[{{ monitor_ipv6_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/is-icmp-packet-lost -a {{monitor_ipv6_test}}
filename = {{ monitor_ipv6_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
{% set monitor_ipv4_test = slave_instance.get('monitor-ipv4-test', '') %}
{% if monitor_ipv4_test %}
{% set monitor_ipv4_section_title = 'check-%s-ipv4-packet-list-test' % slave_instance.get('slave_reference') %}
{% do part_list.append(monitor_ipv4_section_title) -%}
[{{ monitor_ipv4_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/is-icmp-packet-lost -4 -a {{monitor_ipv4_test}}
filename = {{ monitor_ipv4_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
{% set re6st_optimal_test = slave_instance.get('re6st-optimal-test', '') %}
{% if re6st_optimal_test %}
{% set re6st_ipv6, re6st_ipv4 = re6st_optimal_test.split(",") %}
{% set re6st_optimal_test_section_title = 'check-%s-re6st-optimal-test' % slave_instance.get('slave_reference') %}
{% do part_list.append(re6st_optimal_test_section_title) -%}
[{{ re6st_optimal_test_section_title }}]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/check-re6st-optimal-status -4 {{re6st_ipv4}} -6 {{re6st_ipv6}}
filename = {{ re6st_optimal_test_section_title }}
wrapper-path = {{ promise_directory }}/${:filename}
{% endif %}
{# ############################### #}
{# Publish Slave Information #}
{% if not extra_slave_instance_list -%}
{% set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
{% do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
{% for key, value in slave_publish_dict.iteritems() %}
{{ key }} = {{ value }}
{% endfor %}
{% else -%}
{% do slave_instance_information_list.append(slave_publish_dict) -%}
{% endif -%}
{# End of the main for loop#}
{% endfor -%}
###############################################
### Prepare virtualhost for slaves using cache
{% for slave_reference, slave_configuration_section_name in cached_server_dict.iteritems() %}
{% set cached_slave_configuration_section_title = '%s-cached-virtualhost' % slave_reference %}
{% do part_list.append(cached_slave_configuration_section_title) -%}
[{{ cached_slave_configuration_section_title }}]
< = jinja2-template-base
template = {{ template_cached_slave_configuration }}
filename = {{ '%s.conf' % slave_reference }}
rendered = {{ apache_cached_configuration_directory }}/${:filename}
extensions = jinja2.ext.do
extra-context =
section slave_parameter {{ slave_configuration_section_name }}
raw cached_port {{ cached_port }}
raw ssl_cached_port {{ ssl_cached_port }}
{{ '\n' }}
{% endfor %}
[slave-log-directories]
recipe = slapos.cookbook:mkdirectory
{# Define log access #}
[apache-log-access]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directories
raw apache_log_directory {{apache_log_directory}}
raw apache_configuration_directory {{apache_configuration_directory}}
{# Publish information for the instance #}
[publish-apache-information]
recipe = slapos.cookbook:publish
public-ipv4 = {{ public_ipv4 }}
private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list -%}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
{% endif -%}
monitor-base-url = {{ monitor_base_url }}
[buildout]
parts +=
slave-log-directories
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor %}
publish-apache-information
apache-log-access
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
cache-access = {{ cache_access }}
{% endif -%}
apache-graceful-script.sh.in 0000664 0000000 0000000 00000001230 13712426147 0035436 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates #!/bin/sh
RUN_DIR={{ directory_run }}
ETC_DIR={{ directory_etc }}
APACHE_SIGNATURE_FILE=$RUN_DIR/apache_configuration.signature
NAPACHE_SIGNATURE_FILE=$RUN_DIR/napache_configuration.signature
touch $APACHE_SIGNATURE_FILE
sha256sum $ETC_DIR/apache*.conf $ETC_DIR/apache-*.d/*.conf $ETC_DIR/apache-*.d/ssl/*.*key $ETC_DIR/apache-*.d/ssl/*.*crt* | sort -k 66 > $NAPACHE_SIGNATURE_FILE
# If no diff, no restart for now
if diff "$APACHE_SIGNATURE_FILE" "$NAPACHE_SIGNATURE_FILE"; then
echo "Nothing Changed, so nothing to reload"
exit 0
fi
echo "Reloading apache.."
{{ apache_graceful_reload_command }}
mv "$NAPACHE_SIGNATURE_FILE" "$APACHE_SIGNATURE_FILE"
apache-lazy-script-call.sh.in 0000664 0000000 0000000 00000000522 13712426147 0035541 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates #!/bin/bash
PID=$$
PIDFILE={{ pid_file }}
sleep $((RANDOM % 10))
if [ ! -f $PIDFILE ]; then
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
else
ps --pid `cat $PIDFILE` &>/dev/null
if [ $? -eq 0 ]; then
echo "Skipped"
else
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
fi
fi
apache.conf.in 0000664 0000000 0000000 00000016142 13712426147 0032671 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates # Apache configuration file for Zope
# Automatically generated
# Basic server configuration
PidFile "{{ pid_file }}"
ServerName {{ domain }}
DocumentRoot {{ document_root }}
ServerRoot {{ instance_home }}
{{ "Listen %s:%s" % (ipv4_addr, cached_port) }}
{{ "Listen %s:%s" % (ipv4_addr, ssl_cached_port) }}
{% for ip in (ipv4_addr, "[%s]" % ipv6_addr) -%}
{% for port in (http_port, https_port) -%}
{{ "Listen %s:%s" % (ip, port) }}
{% endfor -%}
{% endfor -%}
ServerAdmin {{ server_admin }}
TypesConfig {{ httpd_home }}/conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
ServerTokens Prod
# Disable TRACE Method
TraceEnable off
# Log configuration
ErrorLog "{{ error_log }}"
LogLevel notice
# LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
# LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b" common
# CustomLog "{{ access_log }}" common
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ access_log }}" combined
Order Deny,Allow
Allow from {{ access_control_string }}
Require all granted
Options -Indexes
ErrorDocument 404 /notfound.html
RewriteEngine on
RewriteRule ^/?$ notfound.html [R=404,L]
# List of modules
LoadModule unixd_module {{ httpd_home }}/modules/mod_unixd.so
LoadModule access_compat_module {{ httpd_home }}/modules/mod_access_compat.so
LoadModule authz_core_module {{ httpd_home }}/modules/mod_authz_core.so
LoadModule authz_host_module {{ httpd_home }}/modules/mod_authz_host.so
LoadModule authn_core_module {{ httpd_home }}/modules/mod_authn_core.so
LoadModule log_config_module {{ httpd_home }}/modules/mod_log_config.so
LoadModule deflate_module {{ httpd_home }}/modules/mod_deflate.so
LoadModule setenvif_module {{ httpd_home }}/modules/mod_setenvif.so
LoadModule version_module {{ httpd_home }}/modules/mod_version.so
LoadModule proxy_module {{ httpd_home }}/modules/mod_proxy.so
LoadModule proxy_http_module {{ httpd_home }}/modules/mod_proxy_http.so
LoadModule ssl_module {{ httpd_home }}/modules/mod_ssl.so
LoadModule mime_module {{ httpd_home }}/modules/mod_mime.so
LoadModule dav_module {{ httpd_home }}/modules/mod_dav.so
LoadModule dav_fs_module {{ httpd_home }}/modules/mod_dav_fs.so
LoadModule negotiation_module {{ httpd_home }}/modules/mod_negotiation.so
LoadModule rewrite_module {{ httpd_home }}/modules/mod_rewrite.so
LoadModule headers_module {{ httpd_home }}/modules/mod_headers.so
LoadModule cache_module {{ httpd_home }}/modules/mod_cache.so
LoadModule cache_socache_module {{ httpd_home }}/modules/mod_cache_socache.so
LoadModule socache_shmcb_module {{ httpd_home }}/modules/mod_socache_shmcb.so
LoadModule antiloris_module {{ httpd_home }}/modules/mod_antiloris.so
LoadModule alias_module {{ httpd_home }}/modules/mod_alias.so
LoadModule autoindex_module {{ httpd_home }}/modules/mod_autoindex.so
LoadModule auth_basic_module {{ httpd_home }}/modules/mod_auth_basic.so
LoadModule authz_user_module {{ httpd_home }}/modules/mod_authz_user.so
LoadModule authn_file_module {{ httpd_home }}/modules/mod_authn_file.so
LoadModule filter_module {{ httpd_home }}/modules/mod_filter.so
LoadModule http2_module {{ httpd_home }}/modules/mod_http2.so
LoadModule info_module {{ httpd_home }}/modules/mod_info.so
LoadModule status_module {{ httpd_home }}/modules/mod_status.so
LoadModule reqtimeout_module {{ httpd_home }}/modules/mod_reqtimeout.so
# The following directives modify normal HTTP response behavior to
# handle known problems with browser implementations.
BrowserMatch "Mozilla/2" nokeepalive
BrowserMatch ".*MSIE.*" nokeepalive ssl-unclean-shutdown \
downgrade-1.0 force-response-1.0
BrowserMatch "RealPlayer 4\.0" force-response-1.0
BrowserMatch "Java/1\.0" force-response-1.0
BrowserMatch "JDK/1\.0" force-response-1.0
# The following directive disables redirects on non-GET requests for
# a directory that does not include the trailing slash. This fixes a
# problem with Microsoft WebFolders which does not appropriately handle
# redirects for folders with DAV methods.
# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV.
BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully
BrowserMatch "MS FrontPage" redirect-carefully
BrowserMatch "^WebDrive" redirect-carefully
BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully
BrowserMatch "^gnome-vfs" redirect-carefully
BrowserMatch "^XML Spy" redirect-carefully
BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
{% if slapparameter_dict.get('enable-http2-by-default', 'true') == 'true' %}
Protocols h2 http/1.1
{% endif %}
# Increase IPReadLimit to 10
# IPReadLimit - Maximum simultaneous connections in READ state per IP address
IPReadLimit {{ slapparameter_dict.get('ip-read-limit', '10') }}
ExtendedStatus On
SetHandler server-status
Order Deny,Allow
Deny from all
Allow from All
AuthType basic
AuthName "Apache Server Status"
AuthBasicProvider file
AuthUserFile {{ instance_home }}/etc/monitor-htpasswd
Require valid-user
ServerLimit {{ slapparameter_dict.get('mpm-server-limit', '16') }}
MaxClients {{ slapparameter_dict.get('mpm-max-clients', '400') }}
StartServers {{ slapparameter_dict.get('mpm-start-servers', '3') }}
ThreadsPerChild {{ slapparameter_dict.get('mpm-thread-per-child', '25') }}
#MaxRequestsPerChild 0 // Default value is 0
GracefulShutdownTimeout {{ slapparameter_dict.get('mpm-graceful-shutdown-timeout', '5') }}
# Deflate
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/json application/x-javascript application/javascript application/wasm
BrowserMatch ^Mozilla/4 gzip-only-text/html
BrowserMatch ^Mozilla/4\.0[678] no-gzip
BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
# SSL Configuration
SSLCertificateFile {{ login_certificate }}
SSLCertificateKeyFile {{ login_key }}
{% if slapparameter_dict.get('apache-ca-certificate') %}
SSLCACertificateFile {{ login_ca_crt }}
{% endif %}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)
SSLSessionCacheTimeout 300
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLOptions +StdEnvVars
# Accept proxy to sites using self-signed SSL certificates
SSLProxyCheckPeerCN off
SSLProxyCheckPeerExpire off
include {{frontend_configuration.get('log-access-configuration')}}
includeoptional {{ slave_configuration_directory }}/*.conf
includeoptional {{ slave_with_cache_configuration_directory }}/*.conf
ErrorDocument 404 /notfound.html
RewriteRule (.*) /notfound.html [R=404,L]
cached-virtualhost.conf.in 0000664 0000000 0000000 00000003066 13712426147 0035242 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{% set server_alias_list = slave_parameter.get('server-alias', '').split() %}
{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
# Only accept generic (i.e not Zope) backends on http
ServerName {{ slave_parameter.get('custom_domain') }}
{%- for server_alias in server_alias_list %}
ServerAlias {{ server_alias }}
{% endfor %}
SSLProxyEngine on
{% if ssl_proxy_verify -%}
{% if 'ssl_proxy_ca_crt' in slave_parameter -%}
SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
{% endif %}
SSLProxyVerify require
#SSLProxyCheckPeerCN on
SSLProxyCheckPeerExpire on
{% endif %}
# Rewrite part
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
RewriteRule ^/(.*)$ {{ slave_parameter.get('backend_url', '') }}/$1 [L,P]
ServerName {{ slave_parameter.get('custom_domain') }}
{%- for server_alias in server_alias_list %}
ServerAlias {{ server_alias }}
{% endfor %}
SSLProxyEngine on
{% if ssl_proxy_verify -%}
{% if 'ssl_proxy_ca_crt' in slave_parameter -%}
SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
{% endif %}
SSLProxyVerify require
#SSLProxyCheckPeerCN on
SSLProxyCheckPeerExpire on
{% endif %}
# Rewrite part
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
RewriteRule ^/(.*)$ {{ slave_parameter.get('https_backend_url', '') }}/$1 [L,P]
custom-virtualhost.conf.in 0000664 0000000 0000000 00000000311 13712426147 0035333 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates
{{ slave_parameter.get('apache_custom_https', '') }}
{{ slave_parameter.get('apache_custom_http', '') }}
default-virtualhost.conf.in 0000664 0000000 0000000 00000015406 13712426147 0035460 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{%- set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES -%}
{%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES -%}
{%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES -%}
{%- set server_alias_list = slave_parameter.get('server-alias', '').split() -%}
{%- set enable_h2 = ('' ~ slave_parameter.get('enable-http2', 'true')).lower() in TRUE_VALUES -%}
{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
{%- set disabled_cookie_list = slave_parameter.get('disabled-cookie-list', '').split() -%}
{%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES -%}
{%- set slave_type = slave_parameter.get('type', '') -%}
{%- set ssl_configuration_list = [('SSLCertificateFile', 'path_to_ssl_crt'),
('SSLCertificateKeyFile', 'path_to_ssl_key'),
('SSLCACertificateFile', 'path_to_ssl_ca_crt'),
('SSLCertificateChainFile', 'path_to_ssl_ca_crt')] -%}
ServerName {{ slave_parameter.get('custom_domain') }}
ServerAlias {{ slave_parameter.get('custom_domain') }}
{%- for server_alias in server_alias_list %}
ServerAlias {{ server_alias }}
{% endfor %}
SSLEngine on
SSLProxyEngine on
{% if ssl_proxy_verify -%}
{% if 'ssl_proxy_ca_crt' in slave_parameter -%}
SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
{% endif %}
SSLProxyVerify require
#SSLProxyCheckPeerCN on
SSLProxyCheckPeerExpire on
{% endif %}
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
{% if enable_h2 %}
Protocols h2 http/1.1
{% endif -%}
{% for key, value in ssl_configuration_list -%}
{% if value in slave_parameter -%}
{{ ' %s' % key }} {{ slave_parameter.get(value) }}
{% endif -%}
{% endfor -%}
# One Slave two logs
ErrorLog "{{ slave_parameter.get('error_log') }}"
LogLevel notice
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ slave_parameter.get('access_log') }}" combined
# Rewrite part
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
{% if disable_via_header %}
Header unset Via
{% endif -%}
{% if disable_no_cache_header %}
RequestHeader unset Cache-Control
RequestHeader unset Pragma
{% endif -%}
{%- for disabled_cookie in disabled_cookie_list %}
{{' RequestHeader edit Cookie "(^%(disabled_cookie)s=[^;]*; |; %(disabled_cookie)s=[^;]*|^%(disabled_cookie)s=[^;]*$)" ""' % dict(disabled_cookie=disabled_cookie) }}
{% endfor -%}
{%- if prefer_gzip %}
RequestHeader edit Accept-Encoding "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" "gzip"
{% endif %}
{% if slave_type == 'zope' -%}
{% if 'default-path' in slave_parameter %}
RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
{% endif -%}
# First, we check if we have a zope backend server
# If so, let's use Virtual Host Monster rewrite
# We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', slave_parameter.get('url', '')) }}/VirtualHostBase/https//%{SERVER_NAME}:{{ slave_parameter.get('virtualhostroot-https-port', '443') }}/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
{% elif slave_type == 'redirect' -%}
RewriteRule (.*) {{ slave_parameter.get('https-url', slave_parameter.get('url', ''))}}$1 [R,L]
{% else -%}
{% if 'default-path' in slave_parameter %}
RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
{% endif -%}
RewriteRule ^/(.*)$ {{ slave_parameter.get('https-url', slave_parameter.get('url', '')) }}/$1 [L,P]
{% endif -%}
ServerName {{ slave_parameter.get('custom_domain') }}
ServerAlias {{ slave_parameter.get('custom_domain') }}
{%- for server_alias in server_alias_list %}
ServerAlias {{ server_alias }}
{% endfor %}
SSLProxyEngine on
{% if ssl_proxy_verify -%}
{% if 'ssl_proxy_ca_crt' in slave_parameter -%}
SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
{% endif %}
SSLProxyVerify require
#SSLProxyCheckPeerCN on
SSLProxyCheckPeerExpire on
{% endif %}
# Rewrite part
ProxyPreserveHost On
ProxyTimeout 600
{% if disable_via_header %}
Header unset Via
{% endif -%}
RewriteEngine On
# One Slave two logs
ErrorLog "{{ slave_parameter.get('error_log') }}"
LogLevel notice
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ slave_parameter.get('access_log') }}" combined
{% if enable_h2 %}
Protocols h2 http/1.1
{% endif -%}
{% if disable_no_cache_header %}
RequestHeader unset Cache-Control
RequestHeader unset Pragma
{% endif -%}
{%- for disabled_cookie in disabled_cookie_list %}
{{' RequestHeader edit Cookie "(^%(disabled_cookie)s=[^;]*; |; %(disabled_cookie)s=[^;]*|^%(disabled_cookie)s=[^;]*$)" ""' % dict(disabled_cookie=disabled_cookie) }}
{% endfor -%}
{%- if prefer_gzip %}
RequestHeader edit Accept-Encoding "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" "gzip"
{% endif %}
# Next line is forbidden and people who copy it will be hanged short
{% if https_only -%}
# Not using HTTPS? Ask that guy over there.
# Dummy redirection to https. Note: will work only if https listens
# on standard port (443).
RewriteCond %{SERVER_PORT} !^{{ https_port }}$
RewriteRule ^/(.*) https://%{SERVER_NAME}/$1 [NC,R,L]
{% elif slave_type == 'redirect' -%}
RewriteRule (.*) {{slave_parameter.get('url', '')}}$1 [R,L]
{% elif slave_type == 'zope' -%}
{% if 'default-path' in slave_parameter %}
RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
{% endif -%}
# First, we check if we have a zope backend server
# If so, let's use Virtual Host Daemon rewrite
# We suppose that Apache listens to 80 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/http//%{SERVER_NAME}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
{% else -%}
{% if 'default-path' in slave_parameter %}
RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
{% endif -%}
RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/$1 [L,P]
{% endif -%}
# If nothing exist : put a nice error
# ErrorDocument 404 /notfound.html
# Dadiboom
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates/empty.in0000664 0000000 0000000 00000000016 13712426147 0031732 0 ustar 00root root 0000000 0000000 {{ content }}
nginx-eventsource-slave.conf.in 0000664 0000000 0000000 00000007462 13712426147 0036250 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% set url = slave_parameter.get('url') -%}
{% set https_url = slave_parameter.get('https-url', url) -%}
{% if url.startswith("http://") or url.startswith("https://") -%}
{% set upstream = url.split("/")[2] -%}
{% set https_upstream = https_url.split("/")[2] -%}
{% set protocol = url.split("/")[0] -%}
{% set https_protocol = https_url.split("/")[0] -%}
{% set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) -%}
{% set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) -%}
{%- set ssl_configuration_list = [('ssl_certificate', 'path_to_ssl_crt'),
('ssl_certificate_key', 'path_to_ssl_key')] -%}
upstream {{ slave_parameter.get('slave_reference') }} {
server {{ upstream }};
}
upstream https_{{ slave_parameter.get('slave_reference') }} {
server {{ https_upstream }};
}
server {
listen [{{ global_ipv6 }}]:{{ nginx_http_port }};
listen {{ local_ipv4 }}:{{ nginx_http_port }};
server_name {{ slave_parameter.get('custom_domain') }};
error_log {{ slave_parameter.get('error_log') }} error;
access_log {{ slave_parameter.get('access_log') }} custom;
location /pub {
push_stream_publisher;
push_stream_channels_path $arg_id;
# store messages in memory
push_stream_store_messages off;
# Message size limit
# client_max_body_size MUST be equal to client_body_buffer_size or
# you will be sorry.
client_max_body_size 16k;
client_body_buffer_size 16k;
}
location ~ /sub/(.*) {
# activate subscriber mode for this location
add_header "Access-Control-Allow-Origin" "*";
add_header 'Access-Control-Allow-Credentials' 'false';
add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
push_stream_subscriber eventsource;
# positional channel path
push_stream_channels_path $1;
# content-type
default_type "text/event-stream; charset=utf-8";
}
}
server {
listen [{{ global_ipv6 }}]:{{ nginx_https_port }} ssl;
listen {{ local_ipv4 }}:{{ nginx_https_port }} ssl;
server_name {{ slave_parameter.get('custom_domain') }};
error_log {{ slave_parameter.get('error_log') }} error;
access_log {{ slave_parameter.get('access_log') }} custom;
ssl on;
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
{% for key, value in ssl_configuration_list -%}
{% if value in slave_parameter -%}
{{ ' %s' % key }} {{ slave_parameter.get(value) }};
{% endif -%}
{% endfor %}
location /pub {
push_stream_publisher;
push_stream_channels_path $arg_id;
# store messages in memory
push_stream_store_messages off;
# Message size limit
# client_max_body_size MUST be equal to client_body_buffer_size or
# you will be sorry.
client_max_body_size 16k;
client_body_buffer_size 16k;
}
location ~ /sub/(.*) {
# activate subscriber mode for this location
add_header "Access-Control-Allow-Origin" "*";
add_header 'Access-Control-Allow-Credentials' 'false';
add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
push_stream_subscriber eventsource;
# positional channel path
push_stream_channels_path $1;
# content-type
default_type "text/event-stream; charset=utf-8";
}
}
{% endif -%}
nginx-notebook-slave.conf.in 0000664 0000000 0000000 00000006647 13712426147 0035532 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% set url = slave_parameter.get('url') -%}
{% set https_url = slave_parameter.get('https-url', url) -%}
{% if url.startswith("http://") or url.startswith("https://") -%}
{% set upstream = url.split("/")[2] -%}
{% set https_upstream = https_url.split("/")[2] -%}
{% set protocol = url.split("/")[0] -%}
{% set https_protocol = https_url.split("/")[0] -%}
{% set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) -%}
{% set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) -%}
{%- set ssl_configuration_list = [('ssl_certificate', 'path_to_ssl_crt'),
('ssl_certificate_key', 'path_to_ssl_key')] -%}
upstream {{ slave_parameter.get('slave_reference') }} {
server {{ upstream }};
}
upstream https_{{ slave_parameter.get('slave_reference') }} {
server {{ https_upstream }};
}
server {
listen [{{ global_ipv6 }}]:{{ nginx_http_port }};
listen {{ local_ipv4 }}:{{ nginx_http_port }};
server_name {{ slave_parameter.get('custom_domain') }};
error_log {{ slave_parameter.get('error_log') }} error;
access_log {{ slave_parameter.get('access_log') }} custom;
location / {
proxy_pass {{ proxy_pass }};
proxy_set_header Host $host;
}
location ~ /api/kernels/ {
proxy_pass {{ proxy_pass }};
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
}
location ~ /terminals/ {
proxy_pass {{ proxy_pass }};
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
}
}
server {
listen [{{ global_ipv6 }}]:{{ nginx_https_port }} ssl;
listen {{ local_ipv4 }}:{{ nginx_https_port }} ssl;
server_name {{ slave_parameter.get('custom_domain') }};
error_log {{ slave_parameter.get('error_log') }} error;
access_log {{ slave_parameter.get('access_log') }} custom;
ssl on;
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
{% for key, value in ssl_configuration_list -%}
{% if value in slave_parameter -%}
{{ ' %s' % key }} {{ slave_parameter.get(value) }};
{% endif -%}
{% endfor %}
location / {
proxy_pass {{ https_proxy_pass }};
proxy_set_header Host $host;
}
location ~ /api/kernels/ {
proxy_pass {{ https_proxy_pass }};
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
}
location ~ /terminals/ {
proxy_pass {{ https_proxy_pass }};
proxy_set_header Host $host;
# websocket support
proxy_http_version 1.1;
proxy_set_header Upgrade "websocket";
proxy_set_header Connection "Upgrade";
proxy_read_timeout 86400;
}
}
{% endif -%}
nginx.cfg.in 0000664 0000000 0000000 00000004443 13712426147 0032406 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates daemon off; # run in the foreground so supervisord can look after it
worker_processes {{ worker_processes }};
pid {{ pidfile }};
events {
worker_connections {{ worker_connections }};
# multi_accept on;
}
error_log {{ error_log }};
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
error_log {{ error_log }};
log_format custom '$remote_addr - $remote_user $time_local $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time';
access_log {{ access_log }} custom;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
default_type application/octet-stream;
ssl_certificate {{ ssl_certificate }};
ssl_certificate_key {{ ssl_key }};
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
##
# Push stream Settings
##
push_stream_shared_memory_size 32m;
fastcgi_temp_path {{ varnginx }} 1 2;
uwsgi_temp_path {{ varnginx }} 1 2;
scgi_temp_path {{ varnginx }} 1 2;
client_body_temp_path {{ varnginx }} 1 2;
proxy_temp_path {{ varnginx }} 1 2;
include {{ slave_configuration_directory }}/*.conf;
server {
listen [{{ ip }}]:{{ plain_port }};
listen {{ local_ip }}:{{ plain_port }};
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location / {
root {{ document_root }};
index notfound.html;
}
}
server {
listen [{{ ip }}]:{{ port }} ssl;
listen {{ local_ip }}:{{ port }} ssl;
ssl on;
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
ssl_prefer_server_ciphers on;
ssl_session_cache shared:SSL:10m;
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location / {
root {{ document_root }};
index notfound.html;
}
}
}
notfound.html 0000664 0000000 0000000 00000000327 13712426147 0032714 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates
Instance not found
This instance has not been found.
If this error persists, please check your instance URL and status on SlapOS Master.
replicate-publish-slave-information.cfg.in 0000664 0000000 0000000 00000003557 13712426147 0040337 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% set part_list = [] -%}
{% set slave_information_dict = {} -%}
# regroup slave information from all frontends
{%- for frontend, slave_list_raw in slave_information.iteritems() -%}
{% if slave_list_raw %}
{% set slave_list = json_module.loads(slave_list_raw) -%}
{% else %}
{% set slave_list = [] %}
{% endif %}
{% for slave_dict in slave_list -%}
{% set slave_reference = slave_dict.pop('slave-reference') %}
{% set log_access_url = slave_dict.pop('log-access', '') %}
{% set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
{% do current_slave_dict.update(slave_dict) -%}
{% set log_access_list = current_slave_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend + ': ' + log_access_url) %}
{% do current_slave_dict.__setitem__(
'log-access-urls',
log_access_list
) %}
{% do current_slave_dict.__setitem__(
'replication_number',
current_slave_dict.get('replication_number', 0) + 1
) -%}
{% do slave_information_dict.__setitem__(slave_reference, current_slave_dict) -%}
{% endfor -%}
{% endfor %}
# Publish information for each slave
{% for slave_reference, slave_information in slave_information_dict.iteritems() %}
{% set publish_section_title = 'publish-%s' % slave_reference -%}
{% do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
log-access-url = {{ json_module.dumps(slave_information.pop('log-access-urls', 1000)) }}
{% for key, value in slave_information.iteritems() -%}
{{ key }} = {{ value }}
{% endfor -%}
{% endfor %}
[buildout]
parts =
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
template-log-access.conf.in 0000664 0000000 0000000 00000000724 13712426147 0035300 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates {% for slave, directory in slave_log_directory.iteritems() %}
Alias /{{slave}}/ {{directory}}/
Order Deny,Allow
Deny from env=AUTHREQUIRED
Order Allow,Deny
Deny from all
AuthType Basic
AuthName "Log Access {{slave}}"
AuthUserFile "{{ apache_configuration_directory + '/.' + slave + '.htaccess'}}"
Require user {{slave}}
Options Indexes FollowSymLinks
Satisfy all
{% endfor %}
trafficserver/ 0000775 0000000 0000000 00000000000 13712426147 0033035 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates records.config.jinja2 0000664 0000000 0000000 00000074615 13712426147 0037056 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates/trafficserver #
#
# Process Records Config File
#
#
#
# RECORD-TYPE: CONFIG, LOCAL
# NAME: name of variable
# TYPE: INT, STRING, FLOAT
# VALUE: Initial value for record
#
#
# *NOTE*: All options covered in this file should be documented in the
# administration guide or the addendum:
#
#
##############################################################################
#
# System Variables
#
##############################################################################
CONFIG proxy.config.proxy_name STRING {{ ats_configuration['hostname'] }}
CONFIG proxy.config.local_state_dir {{ ats_directory['local-state'] }}
CONFIG proxy.config.config_dir STRING {{ ats_directory['configuration'] }}
CONFIG proxy.config.bin_path STRING {{ ats_directory['bin_path'] }}
CONFIG proxy.config.proxy_binary_opts STRING -M
CONFIG proxy.config.env_prep STRING example_prep.sh
CONFIG proxy.config.alarm_email STRING nobody
CONFIG proxy.config.syslog_facility STRING LOG_DAEMON
CONFIG proxy.config.output.logfile STRING traffic.out
CONFIG proxy.config.snapshot_dir STRING snapshots
CONFIG proxy.config.system.mmap_max INT 2097152
##############################################################################
#
# Main threads configuration (worker threads). Also see configurations for
# SSL threads, disk I/O threads and task threads in their respective areas.
#
##############################################################################
CONFIG proxy.config.exec_thread.autoconfig INT 1
CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.5
CONFIG proxy.config.exec_thread.limit INT 2
CONFIG proxy.config.accept_threads INT 1
##############################################################################
#
# Local Manager
#
##############################################################################
CONFIG proxy.config.admin.admin_user STRING admin
CONFIG proxy.config.admin.number_config_bak INT 3
CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
##############################################################################
#
# Process Manager
#
##############################################################################
CONFIG proxy.config.admin.autoconf_port INT {{ ats_configuration.get('autoconf-port', '8083') }}
CONFIG proxy.config.process_manager.mgmt_port INT {{ ats_configuration.get('mgmt-port', '8084') }}
##############################################################################
#
# In order to only bind a specific IP, use the following config, as in
# the example below. Note - this can contain two addresses, one for IPv4
# sockets and one for IPv6 sockets.
#
##############################################################################
LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
#LOCAL proxy.local.outgoing_ip_to_bind STRING 10.0.143.32
#LOCAL proxy.local.incoming_ip_to_bind STRING 192.168.101.17 fc07:192:168:101::17
##############################################################################
#
# Alarm Configuration
#
##############################################################################
# execute alarm as "/ """
CONFIG proxy.config.alarm.bin STRING example_alarm_bin.sh
CONFIG proxy.config.alarm.abs_path STRING NULL
##############################################################################
#
# HTTP Engine
#
##############################################################################
##########
# basics #
##########
# The server ports are listed here. These are separated by spaces or commas.
# Each port is a colon separated list of values, which must include a
# port number. The order is irrelevant. Other options are
# ipv4 - Use IPv4 (default)
# ipv6 - Use IPv6
# tr-in - Transparent inbound.
# tr-out - Transparent outbound.
# tr-full - Fully transparent (inbound and outbound).
# tr-pass - Transparently Pass-through non-HTTP traffic (in conjuction with tr-in).
# ssl - SSL terminated port.
# blind - Blind tunnel port (CONNECT only)
# ip-in=[addr] - Bind inbound IP address (listen for client).
# ip-out=[addr] - Bind outbound IP address (connect to origin server).
# ip-resolve=[style] - Set the IP resolution style.
#
# Note - address types must agree with each other and the ipv4/ipv6
# option if specified. IPv6 addresses must be enclosed in brackets.
# ip-out can be repeated as long as each address is a different
# family. If ip-in is specified as an IPv6 address, the port is
# forced to IPv6. Transparent ports cannot be bound to an IP
# address on the transparent side.
#
# The '=' is optional for any option with a value.
#
# Example 1: Port 8080 IPv6 inbound transparent, and port 80 IPv4
# "8080:ipv6:tr-in 80"
#
# Example 2: Listen on standard http and https ports for IPv4 and IPv6,
# fully transparent on the http ports. Also provide an non-transparent
# port at address 192.168.1.56 on port 8080.
# "80:ipv4:tr-full tr-full:80:ipv6 443:ipv4:ssl 443:ssl:ipv6 ip-in=192.168.1.56:8080"
#
CONFIG proxy.config.http.server_ports STRING {{ ats_configuration['local-ip'] + ':' + ats_configuration['input-port'] }}
# Ports on the origin server to which a blind tunnel may connect.
CONFIG proxy.config.http.connect_ports STRING 443 563
# The via settings have four values
# 0 - Do not modify / set this via header
# 1 - Update the via, with normal verbosity
# 2 - Update the via, with higher verbosity
# 3 - Update the via, with highest verbosity
CONFIG proxy.config.http.insert_request_via_str INT 2
CONFIG proxy.config.http.insert_response_via_str INT 2
# Insert a Server: header, this has three values
# 0 - Don't add or modify the Server: header
# 1 - Add a Server: header
# 2 - Only add a Server: header if one doesn't exist already
CONFIG proxy.config.http.response_server_enabled INT 1
CONFIG proxy.config.http.insert_age_in_response INT 1
CONFIG proxy.config.http.enable_url_expandomatic INT 0
CONFIG proxy.config.http.no_dns_just_forward_to_parent INT 0
CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
CONFIG proxy.config.http.keep_alive_enabled_in INT 1
CONFIG proxy.config.http.keep_alive_enabled_out INT 1
CONFIG proxy.config.http.chunking_enabled INT 1
# send http11 requests:
# 0 - Never
# 1 - Always
# 2 - if the server has returned http1.1 before
# 3 - if the client request is 1.1 & the server has returned 1.1 before
# If use_client_addr is set to 1, options 2 and 3 cause the proxy to use
# the client HTTP version for upstream requests.
CONFIG proxy.config.http.send_http11_requests INT 1
# Share server connections
# 0 - Never
# 1 - Share, with a single global connection pool
# 2 - Share, with a connection pool per worker thread
CONFIG proxy.config.http.share_server_sessions INT 2
##########################
# HTTP referer filtering #
##########################
CONFIG proxy.config.http.referer_filter INT 0
CONFIG proxy.config.http.referer_format_redirect INT 0
CONFIG proxy.config.http.referer_default_redirect STRING http://www.example.com/
##############################
# parent proxy configuration #
##############################
CONFIG proxy.config.http.parent_proxy_routing_enable INT 0
CONFIG proxy.config.http.parent_proxy.retry_time INT 300
# Parent fail threshold is the number of request that must
# fail within the retry window for the parent to be marked
# down
CONFIG proxy.config.http.parent_proxy.fail_threshold INT 10
CONFIG proxy.config.http.parent_proxy.total_connect_attempts INT 4
CONFIG proxy.config.http.parent_proxy.per_parent_connect_attempts INT 2
CONFIG proxy.config.http.parent_proxy.connect_attempts_timeout INT 30
CONFIG proxy.config.http.forward.proxy_auth_to_parent INT 0
###################################
# HTTP connection timeouts (secs) #
###################################
# out: proxy -> origin server connection
# in : ua -> proxy connection
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 115
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 300
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 300
CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120
CONFIG proxy.config.http.background_fill_active_timeout INT 60
CONFIG proxy.config.http.background_fill_completed_threshold FLOAT 0.5
##################################
# origin server connect attempts #
##################################
CONFIG proxy.config.http.connect_attempts_max_retries INT 1
CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 3
CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
CONFIG proxy.config.http.connect_attempts_timeout INT 160
CONFIG proxy.config.http.post_connect_attempts_timeout INT 1800
CONFIG proxy.config.http.down_server.cache_time INT 300
CONFIG proxy.config.http.down_server.abort_threshold INT 10
##################################
# congestion control #
##################################
CONFIG proxy.config.http.congestion_control.enabled INT 0
#############################
# negative response caching #
#############################
CONFIG proxy.config.http.negative_caching_enabled INT 0
CONFIG proxy.config.http.negative_caching_lifetime INT 1800
#########################
# proxy users variables #
#########################
CONFIG proxy.config.http.anonymize_remove_from INT 0
CONFIG proxy.config.http.anonymize_remove_referer INT 0
CONFIG proxy.config.http.anonymize_remove_user_agent INT 0
CONFIG proxy.config.http.anonymize_remove_cookie INT 0
CONFIG proxy.config.http.anonymize_remove_client_ip INT 0
CONFIG proxy.config.http.anonymize_insert_client_ip INT 1
CONFIG proxy.config.http.anonymize_other_header_list STRING NULL
CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 1
############
# security #
############
CONFIG proxy.config.http.push_method_enabled INT 0
# ###################################
# # HTTP Quick filtering (security) #
# ###################################
# this functionality is moved to ip_allow.config
#################
# cache control #
#################
CONFIG proxy.config.http.cache.http INT 1
# Enabling this setting allows the proxy to cache empty documents. This currently
# requires that the response has a Content-Length: header, with a value of "0".
CONFIG proxy.config.http.cache.allow_empty_doc INT 0
CONFIG proxy.config.http.cache.ignore_client_no_cache INT 0
CONFIG proxy.config.http.cache.ims_on_client_no_cache INT 0
CONFIG proxy.config.http.cache.ignore_server_no_cache INT 0
CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 0
CONFIG proxy.config.http.normalize_ae_gzip INT 0
# cache responses to cookies has 5 options:
# 0 - do not cache any responses to cookies
# 1 - cache for any content-type
# 2 - cache only for image types
# 3 - cache for all but text content-types
# 4 - cache for all but text content-types except OS response
# without "Set-Cookie" or with "Cache-Control: public"
# See also cache-responses-to-cookies in cache.config.
CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
CONFIG proxy.config.http.cache.ignore_authentication INT 0
CONFIG proxy.config.http.cache.cache_urls_that_look_dynamic INT 1
CONFIG proxy.config.http.cache.enable_default_vary_headers INT 0
# when_to_revalidate has 5 options:
# 0 - default. use cache directives or heuristic
# 1 - stale if heuristic
# 2 - always stale (always revalidate)
# 3 - never stale
# 4 - always revalidate if request is conditional, else default is used
CONFIG proxy.config.http.cache.when_to_revalidate INT 0
# Some old MSIE browsers don't send no-cache headers to
# reverse proxies or transparent caches, this variable controls
# when to add no-cache headers to MSIE requests:
# -1 - no-cache is never added, stats are not updated
# 0 - default; no-cache not added to MSIE requests
# 1 - no-cache added to IMS MSIE requests
# 2 - no-cache added to all MSIE requests
CONFIG proxy.config.http.cache.when_to_add_no_cache_to_msie_requests INT -1
# required headers: three options:
# 0 - No required headers to make document cachable
# 1 - "Last-Modified:", "Expires:", or "Cache-Control: max-age" required
# 2 - explicit lifetime required, "Expires:" or "Cache-Control: max-age"
CONFIG proxy.config.http.cache.required_headers INT 2
CONFIG proxy.config.http.cache.max_stale_age INT 604800
CONFIG proxy.config.http.cache.range.lookup INT 1
########################
# heuristic expiration #
########################
CONFIG proxy.config.http.cache.heuristic_min_lifetime INT 3600
CONFIG proxy.config.http.cache.heuristic_max_lifetime INT 86400
CONFIG proxy.config.http.cache.heuristic_lm_factor FLOAT 0.10
CONFIG proxy.config.http.cache.fuzz.time INT 240
CONFIG proxy.config.http.cache.fuzz.probability FLOAT 0.005
#########################################
# dynamic content & content negotiation #
#########################################
CONFIG proxy.config.http.cache.vary_default_text STRING NULL
CONFIG proxy.config.http.cache.vary_default_images STRING NULL
CONFIG proxy.config.http.cache.vary_default_other STRING NULL
##############################################################
# The HTTP stats are expensive, turn off you don't need them #
##############################################################
CONFIG proxy.config.http.enable_http_stats INT 1
##############################################################################
#
# Customizable User Response Pages
#
##############################################################################
# 1 - enable customizable user response pages in only the "default" directory
# 2 - enable language-targeted user response pages
CONFIG proxy.config.body_factory.enable_customizations INT 1
CONFIG proxy.config.body_factory.enable_logging INT 0
# 0 - never suppress generated responses
# 1 - always suppress generated responses
# 2 - suppress responses for intercepted traffic
CONFIG proxy.config.body_factory.response_suppression_mode INT 0
##############################################################################
#
# Net Subsystem
#
##############################################################################
CONFIG proxy.config.net.connections_throttle INT 30000
# Enable defer accept / accept filtering. On Linux, this is a timeout, sec.
CONFIG proxy.config.net.defer_accept INT 45
##############################################################################
#
# Cluster Subsystem
#
##############################################################################
# cluster type requires restart to change
# 1 is full clustering, 2 is mgmt only, 3 is no clustering
LOCAL proxy.local.cluster.type INT 3
CONFIG proxy.config.cluster.cluster_port INT 8086
CONFIG proxy.config.cluster.rsport INT 8088
CONFIG proxy.config.cluster.mcport INT 8089
CONFIG proxy.config.cluster.mc_group_addr STRING 224.0.1.37
CONFIG proxy.config.cluster.mc_ttl INT 1
CONFIG proxy.config.cluster.log_bogus_mc_msgs INT 1
CONFIG proxy.config.cluster.ethernet_interface STRING lo
##############################################################################
#
# Cache
#
##############################################################################
CONFIG proxy.config.cache.permit.pinning INT 0
# default the ram cache size to AUTO_SIZE (-1) based on cache size
# (approximately 10 MB of RAM cache per GB of disk cache)
# alternatively, set to a fixed value such as 21474836480 (20GB)
CONFIG proxy.config.cache.ram_cache.size INT {{ ats_configuration.get('ram-cache-size', '1G') }}
# Replacement algorithm
# 0 : Clocked Least Frequently Used by Size (CLFUS) w/optional compression
# 1 : LRU w/o optional compression - trivially simple
CONFIG proxy.config.cache.ram_cache.algorithm INT 0
# Filter inserts into the RAM cache to ensure that they have been seen at
# least once. For LRU, this provides scan resistance. Note that CLFUS
# already requires that a document have history before it is inserted, so
# for CLFUS, setting this option means that a document must be seen three
# times before it is added to the RAM cache.
CONFIG proxy.config.cache.ram_cache.use_seen_filter INT 0
# Compress the content of the ram cache:
# 0 : no compression
# 1 : fastlz (extremely fast, relatively low compression)
# 2 : libz (moderate speed, reasonable compression)
# 3 : liblzma (very slow, high compression)
# NOTE: compression runs on task threads. To use more cores for
# compression, increase proxy.config.task_threads.
CONFIG proxy.config.cache.ram_cache.compress INT 0
# The maximum number of alternates that are allowed for any given URL.
# It is not possible to strictly enforce this if the variable
# 'proxy.config.cache.vary_on_user_agent' is set to 1.
# The default value for 'proxy.config.cache.vary_on_user_agent' is 0.
# (0 disables the maximum number of alts check)
CONFIG proxy.config.cache.limits.http.max_alts INT 5
# The target size of a contiguous fragment on disk.
# Acceptable values are powers of 2, e.g. 65536, 131072, 262144, 524288, 1048576, 2097152.
# Larger could waste memory on slow connections, smaller could waste seeks.
CONFIG proxy.config.cache.target_fragment_size INT 1048576
# The maximum size of a document that will be stored in the cache.
# (0 disables the maximum document size check)
CONFIG proxy.config.cache.max_doc_size INT 0
# enable the cache to read from an object while it is being added to the cache
CONFIG proxy.config.cache.enable_read_while_writer INT 0
# How many I/O threads to allocate per disk (spindle). Be aware that RAID
# disks would show up to TS as a single spindle.
CONFIG proxy.config.cache.threads_per_disk INT 8
# Time (in ms) to delay until retrying to acquire a cache lock. Setting
# this low can reduce latencies in some cases, but can consume more CPU.
# If you experience CPU spinning, try increasing this setting.
CONFIG proxy.config.cache.mutex_retry_delay INT 2
# The interim storage disks. Must use raw disks.
# Only support at most 8 interim disks now. e.g.
# proxy.config.cache.interim.storage STRING /dev/sda /dev/sdb/
LOCAL proxy.config.cache.interim.storage STRING NULL
##############################################################################
#
# DNS
#
##############################################################################
CONFIG proxy.config.dns.search_default_domains INT 0
CONFIG proxy.config.dns.splitDNS.enabled INT 0
CONFIG proxy.config.dns.max_dns_in_flight INT 2048
# Additional URL expansions for http DNS lookup
CONFIG proxy.config.dns.url_expansions STRING NULL
CONFIG proxy.config.dns.round_robin_nameservers INT 0
CONFIG proxy.config.dns.nameservers STRING NULL
CONFIG proxy.config.dns.resolv_conf STRING /etc/resolv.conf
# This provides additional resilience against DNS forgery, particularly in
# forward or transparent proxies, but requires that the resolver populates
# the queries section of the response properly.
CONFIG proxy.config.dns.validate_query_name INT 0
##############################################################################
#
# HostDB
#
##############################################################################
# in entries, may not be changed while running
# note that in order to increase hostdb.size, hostdb.storage_size should
# also be increase. These are best guesses, you will have to monitor this.
CONFIG proxy.config.hostdb.size INT 120000
CONFIG proxy.config.hostdb.storage_size INT 32M
# ttl modes:
# 0 = obey
# 1 = ignore
# 2 = min(X,ttl)
# 3 = max(X,ttl)
CONFIG proxy.config.hostdb.ttl_mode INT 0
# in minutes...
CONFIG proxy.config.hostdb.timeout INT 1440
# round-robin addresses for single clients
# (can cause authentication problems)
CONFIG proxy.config.hostdb.strict_round_robin INT 0
##############################################################################
#
# Logging Config
#
##############################################################################
# possible values for logging_enabled:
# 0: no logging at all
# 1: log errors only
# 2: log transactions only
# 3: full logging (errors + transactions)
CONFIG proxy.config.log.logging_enabled INT 3
CONFIG proxy.config.log.max_secs_per_buffer INT 5
CONFIG proxy.config.log.max_space_mb_for_logs INT 25000
CONFIG proxy.config.log.max_space_mb_for_orphan_logs INT 25
CONFIG proxy.config.log.max_space_mb_headroom INT 1000
CONFIG proxy.config.log.hostname STRING localhost
CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
CONFIG proxy.config.log.logfile_perm STRING rw-r--r--
CONFIG proxy.config.log.custom_logs_enabled INT 0
CONFIG proxy.config.log.squid_log_enabled INT 1
CONFIG proxy.config.log.squid_log_is_ascii INT 0
CONFIG proxy.config.log.squid_log_name STRING squid
CONFIG proxy.config.log.squid_log_header STRING NULL
CONFIG proxy.config.log.common_log_enabled INT 0
CONFIG proxy.config.log.common_log_is_ascii INT 1
CONFIG proxy.config.log.common_log_name STRING common
CONFIG proxy.config.log.common_log_header STRING NULL
CONFIG proxy.config.log.extended_log_enabled INT 0
CONFIG proxy.config.log.extended_log_is_ascii INT 0
CONFIG proxy.config.log.extended_log_name STRING extended
CONFIG proxy.config.log.extended_log_header STRING NULL
CONFIG proxy.config.log.extended2_log_enabled INT 0
CONFIG proxy.config.log.extended2_log_is_ascii INT 1
CONFIG proxy.config.log.extended2_log_name STRING extended2
CONFIG proxy.config.log.extended2_log_header STRING NULL
CONFIG proxy.config.log.separate_icp_logs INT 0
CONFIG proxy.config.log.separate_host_logs INT 0
# Log collation allows you to do "remote logging"
LOCAL proxy.local.log.collation_mode INT 0
CONFIG proxy.config.log.collation_host STRING NULL
CONFIG proxy.config.log.collation_port INT 8085
CONFIG proxy.config.log.collation_secret STRING foobar
CONFIG proxy.config.log.collation_host_tagged INT 0
CONFIG proxy.config.log.collation_retry_sec INT 5
CONFIG proxy.config.log.rolling_enabled INT 1
CONFIG proxy.config.log.rolling_interval_sec INT 86400
CONFIG proxy.config.log.rolling_offset_hr INT 0
CONFIG proxy.config.log.rolling_size_mb INT 10
CONFIG proxy.config.log.auto_delete_rolled_files INT 1
CONFIG proxy.config.log.sampling_frequency INT 1
##############################################################################
#
# Reverse Proxy
#
##############################################################################
CONFIG proxy.config.reverse_proxy.enabled INT 1
CONFIG proxy.config.header.parse.no_host_url_redirect STRING NULL
##############################################################################
#
# URL Remap Rules
#
##############################################################################
CONFIG proxy.config.url_remap.default_to_server_pac INT 0
CONFIG proxy.config.url_remap.default_to_server_pac_port INT -1
# To enable forward proxy, you must turn off remap_required
CONFIG proxy.config.url_remap.remap_required INT 1
# Pristine host header is the "original" (request) header. Make sure your
# origin expects them in reverse proxy.
CONFIG proxy.config.url_remap.pristine_host_hdr INT 1
##############################################################################
#
# SSL Termination
#
##############################################################################
# The number of SSL threads is a multiplier of number of CPUs and
# proxy.config.exec_thread.autoconfig.scale by default. You can
# override that here (set it to a non-zero value).
CONFIG proxy.config.ssl.number.threads INT 0
# The following variables control SSL protocols.
CONFIG proxy.config.ssl.SSLv2 INT 0
CONFIG proxy.config.ssl.SSLv3 INT 0
CONFIG proxy.config.ssl.TLSv1 INT 1
CONFIG proxy.config.ssl.TLSv1_1 INT 1
CONFIG proxy.config.ssl.TLSv1_2 INT 1
# The following two variables control the Cipher Suite traffic Server
# uses for HTTPS connnections and whether to prefer the client
# selected (default) or the server selected
# Our default SSL Cipher Suite tries to be reasonably fast and strong.
CONFIG proxy.config.ssl.server.cipher_suite STRING ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
CONFIG proxy.config.ssl.server.honor_cipher_order INT 1
# Control if SSL should perform content compression or not
CONFIG proxy.config.ssl.compression INT 0
# Client certification level should be:
# 0 no client certificates
# 1 client certificates optional
# 2 client certificates required
CONFIG proxy.config.ssl.client.certification_level INT 0
# Server cert chain filename is the name of the global cert chain file
# that is added to every cert in ssl_multicert.config. This file is only
# loaded if there are configurations in ssl_multicert.config.
CONFIG proxy.config.ssl.server.cert_chain.filename STRING NULL
# This is the path that SSL certificates files are relative to. Certificate
# names specified in ssl_multicert.config will be located relative to this path.
CONFIG proxy.config.ssl.server.cert.path STRING etc/trafficserver
# If any private key is not contained in the certificate file, you must
# fill in the private key path. Private key names specified in
# ssl_multicert.config will be located relative to this path.
CONFIG proxy.config.ssl.server.private_key.path STRING etc/trafficserver
# The CA file name and path are the
# certificate authority certificate that
# client certificates will be verified against.
CONFIG proxy.config.ssl.CA.cert.filename STRING NULL
CONFIG proxy.config.ssl.CA.cert.path STRING etc/trafficserver
################################
# client related configuration #
################################
CONFIG proxy.config.ssl.client.verify.server INT 0
CONFIG proxy.config.ssl.client.cert.filename STRING NULL
CONFIG proxy.config.ssl.client.cert.path STRING etc/trafficserver
# Fill in private key file and path only if the client's
# private key is not contained in the client certificate file.
CONFIG proxy.config.ssl.client.private_key.filename STRING NULL
CONFIG proxy.config.ssl.client.private_key.path STRING etc/trafficserver
# The CA file name and path are the
# certificate authority certificate that
# server certificates will be verified against.
CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL
CONFIG proxy.config.ssl.client.CA.cert.path STRING etc/trafficserver
##############################################################################
#
# ICP Configuration. NOTE! ICP is currently broken NOTE!
#
##############################################################################
# icp modes
# enabled=0 ICP disabled
# enabled=1 Allow receive of ICP queries
# enabled=2 Allow send/receive of ICP queries
CONFIG proxy.config.icp.enabled INT 0
CONFIG proxy.config.icp.icp_interface STRING NULL
CONFIG proxy.config.icp.icp_port INT 3130
CONFIG proxy.config.icp.multicast_enabled INT 0
CONFIG proxy.config.icp.query_timeout INT 2
##############################################################################
#
# Scheduled Update Configuration
#
##############################################################################
CONFIG proxy.config.update.enabled INT 0
CONFIG proxy.config.update.force INT 0
CONFIG proxy.config.update.retry_count INT 10
CONFIG proxy.config.update.retry_interval INT 2
CONFIG proxy.config.update.concurrent_updates INT 100
##############################################################################
#
# Socket send/recv buffer sizes (0 == don't call setsockopt() )
#
##############################################################################
# out: proxy -> os connection
# in : ua -> proxy connection
CONFIG proxy.config.net.sock_send_buffer_size_in INT 262144
CONFIG proxy.config.net.sock_recv_buffer_size_in INT 0
CONFIG proxy.config.net.sock_send_buffer_size_out INT 0
CONFIG proxy.config.net.sock_recv_buffer_size_out INT 0
##############################################################################
#
# User Overridden Configurations Below
#
##############################################################################
CONFIG proxy.config.core_limit INT -1
##############################################################################
#
# Debugging
#
##############################################################################
# Uses a regular expression to match the debugging topic name, performance
# will be affected!
CONFIG proxy.config.diags.debug.enabled INT 0
CONFIG proxy.config.diags.debug.tags STRING http.*|dns.*
# Great for tracking down memory leaks, but you need to use the
# ink allocators
CONFIG proxy.config.dump_mem_info_frequency INT 0
# Log the source code location of diagnostic messages.
CONFIG proxy.config.diags.show_location INT 0
##############################################################################
#
# Configuration for Reclaimable InkFreeList memory pool
#
# NOTE: The following options are meaningfull only when Traffic Server is
# compiled with the following option to configure:
#
# --enable-reclaimable-freelist
#
##############################################################################
CONFIG proxy.config.allocator.enable_reclaim INT 1
# The value of reclaim_factor should be in the 0.0 to 1.0 range. Allocators
# use it to calculate size of unused memory, which is used to determine when
# to reclaim memory. The larger the value, the more aggressive reclaims.
CONFIG proxy.config.allocator.reclaim_factor FLOAT 0.300000
# Allocator will reclaim memory only when it continuously satisfy the reclaim
# condition for max_overage continuous checks.
CONFIG proxy.config.allocator.max_overage INT 3
# For debugging, enable debug_filter, which is a bit-map with these fields:
# bit 0: reclaim memory in ink_freelist_new
# bit 1: allocate memory from partial-free Chunks(if exist) or OS
CONFIG proxy.config.allocator.debug_filter INT 0
##############################################################################
#
# Slow Log
#
##############################################################################
# Log any request that takes more then x number of milliseconds, needs
# to be > 0 to be enabled
CONFIG proxy.config.http.slow.log.threshold INT 0
##############################################################################
#
# Thread pool for "misc" tasks, plugins etc. 2 is a good minimum.
#
##############################################################################
CONFIG proxy.config.task_threads INT 2
storage.config.jinja2 0000664 0000000 0000000 00000000207 13712426147 0037043 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates/trafficserver {{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }} volume={{ ats_configuration.get("hostname") }} wrapper.in 0000664 0000000 0000000 00000000044 13712426147 0032176 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/apache-frontend/templates #!${dash-output:dash}
{{ content }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/ 0000775 0000000 0000000 00000000000 13712426147 0025707 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/buildout.hash.cfg 0000664 0000000 0000000 00000003265 13712426147 0031147 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-nginx-service]
filename = template-nginx-service.sh.in
md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
[template-nginx-configuration]
filename = template-nginx.cfg.in
md5sum = fc4f258e3ae56e0c9596484d736ccfed
[template-dcron-service]
filename = template-dcron-service.sh.in
md5sum = 1372441dac23e4fa7d2dc773a74725ea
[template-backup-script]
filename = template-backup-script.sh.in
md5sum = 4c75f8462eaef2d8fcf24627e5f3575a
[template-crontab-line]
filename = template-crontab-line.in
md5sum = 5cbd64f04da0601ba4286516a6161f5e
[template-crontab]
filename = template-crontab.in
md5sum = f1f82101258de19068262b7213fc478b
[status2rss]
filename = status2rss.py
md5sum = 432d22bb0f67df5203bbc5d1134a952b
[template-update-rss-script]
filename = template-update-rss.sh.in
md5sum = ae4a0043414336a521b524d9c95f1c68
[template-pullrdiffbackup]
filename = instance-pullrdiffbackup.cfg.in
md5sum = c1f00c251298c2ab7fd095e7c4571d3b
[template]
filename = instance.cfg.in
md5sum = 42021b325159dff29e4bd4e33b8ff2f3
instance-pullrdiffbackup.cfg.in 0000664 0000000 0000000 00000020567 13712426147 0033707 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver {% set part_list = [] -%}
{% set crontab_line_list = [] -%}
###############################
#
# Instanciate dcron
#
###############################
[variables]
status_dirbasename = status
statistic_dirbasename = statistic
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
varnginx = $${:var}/nginx
# scripts = $${:etc}/run
services = $${:etc}/service
cron-entries = $${:etc}/cron.d
cron-lines = $${:etc}/cron.lines
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
backup = $${:srv}/backup
status = $${:srv}/$${variables:status_dirbasename}
statistic = $${:srv}/$${variables:statistic_dirbasename}
backupscript = $${:etc}/backup
www = $${:srv}/www
home = $${:etc}/home
ssl = $${:etc}/ssl
ssh = $${:home}/.ssh
plugin = $${:etc}/plugin
#################################
# Cron service
#################################
[dcron-service]
recipe = slapos.recipe.template
url = ${template-dcron-service:output}
output = $${directory:services}/crond
mode = 0700
logfile = $${directory:log}/crond.log
#################################
# Slave backup scripts and crontab
#################################
# Go throught slave list to set their configuration
{% for slave_instance in slave_instance_list -%}
{% set orig_slave_reference = slave_instance.get('slave_reference') -%}
{% set slave_reference = orig_slave_reference.replace(' ', '_') -%}
{% set frequency = slave_instance.get('frequency', '') -%}
{% set hostname = slave_instance.get('hostname', '') -%}
{% set connection = slave_instance.get('connection', '') -%}
{% set connection_port = slave_instance.get('connection_port', '22') -%}
{% set include = slave_instance.get('include', '') -%}
{% set include_string = "' --include='".join(include.split(' ')) -%}
{% set exclude = slave_instance.get('exclude', '') -%}
{% set exclude_string = '' -%}
{% set sudo = slave_instance.get('sudo', 'False') -%}
{% set remote_schema = slave_instance.get('remote_rdiff_path', 'rdiff-backup') + ' --server --restrict-read-only / -- "$@"' -%}
{% if (exclude != '') -%}
{% set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
{% set exclude_string = "--exclude='" + exclude_string + "'" -%}
{% endif -%}
{% if (sudo == 'True') -%}
{% set remote_schema = 'sudo backupagent_rdiff-backup' -%}
{% endif -%}
{% if (frequency != '') and (hostname != '') and (connection != '') and (include != '') -%}
[{{ slave_reference }}-backup-directory]
recipe = slapos.cookbook:mkdirectory
directory = $${directory:backup}/$${:_buildout_section_name_}
[{{ slave_reference }}-backup-private_key]
recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:rm} -f $${:key} $${:public_key} && ${openssh-output:keygen} -t $${:type} -b 2048 -f $${:key} -q -N ""
key = $${directory:ssh}/$${:_buildout_section_name_}
public_key = $${:key}.pub
location = $${:public_key}
type = rsa
# Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run.
{% do part_list.insert(0, "%s-backup-private_key" % slave_reference) -%}
[{{ slave_reference }}-backup-read-public_key]
recipe = slapos.cookbook:readline
storage-path = {{ '$${' ~ slave_reference }}-backup-private_key:public_key}
# Publish slave {{ slave_reference }} information
[{{ slave_reference }}-backup-publish]
recipe = slapos.cookbook:publish
-slave-reference = {{ orig_slave_reference }}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/{{ '$${' ~ slave_reference }}-backup-script:status_name}.rss
{% do part_list.append("%s-backup-publish" % slave_reference) -%}
[{{ slave_reference }}-promise-check-backup]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
output = $${directory:plugin}/{{ slave_reference }}_check_backup.py
content =
from slapos.promise.plugin.backupserver_check_backup import RunPromise
config-status_dirbasename = $${variables:status_dirbasename}
config-status_name = {{ '$${' ~ slave_reference }}-backup-script:status_name}
config-status_fullpath = {{ '$${' ~ slave_reference }}-backup-script:status_log}
config-script_fullpath = {{ '$${' ~ slave_reference }}-backup-script:output}
config-cron_frequency = {{ frequency }}
config-monitor_url = $${monitor-publish:monitor-base-url}
config-statistic_dirbasename = $${variables:statistic_dirbasename}
config-statistic_name = {{ '$${' ~ slave_reference }}-backup-script:statistic_name}
mode = 600
{% do part_list.append("%s-promise-check-backup" % slave_reference) -%}
[{{ slave_reference }}-backup-script]
recipe = slapos.recipe.template
url = ${template-backup-script:output}
output = $${directory:backupscript}/$${:_buildout_section_name_}
mode = 0700
datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
connection = {{ connection }}
connection_port = {{ connection_port }}
hostname = {{ hostname }}
include = {{ include_string }}
exclude_string = {{ exclude_string }}
remote_schema = {{ remote_schema }}
status_name = {{ slave_reference }}_status.txt
statistic_name = {{ slave_reference }}_statistic.txt
status_log = $${directory:status}/$${:status_name}
statistic_log = $${directory:statistic}/$${:statistic_name}
[{{ slave_reference }}-backup-crontab-line]
recipe = slapos.recipe.template
url = ${template-crontab-line:output}
output = $${directory:cron-lines}/$${:_buildout_section_name_}
mode = 0600
script = {{ '$${' ~ slave_reference }}-backup-script:output}
frequency = {{ frequency }}
{% do crontab_line_list.append("$${%s-backup-crontab-line:output}" % slave_reference) -%}
{% endif -%}
{% endfor -%}
#################################
# Generate crontab file
#################################
[update-rss-script]
recipe = slapos.recipe.template
url = ${template-update-rss-script:output}
output = $${directory:etc}/$${:_buildout_section_name_}
mode = 0700
global_rss = $${slap-connection:computer-id}-$${slap-connection:partition-id}.rss
[update-rss-crontab-line]
recipe = slapos.recipe.template
url = ${template-crontab-line:output}
output = $${directory:cron-lines}/$${:_buildout_section_name_}
mode = 0600
script = $${update-rss-script:output}
frequency = */5 * * * *
{% do crontab_line_list.append("$${update-rss-crontab-line:output}") -%}
[publish-global-rss]
recipe = slapos.cookbook:publish
<= monitor-publish
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${update-rss-script:global_rss}
{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
[activate-crontab-file]
# XXX File is never removed
recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = nginx_listen.py
config-hostname = $${nginx-configuration:ip}
config-port = $${nginx-configuration:port}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
ssl_crt = $${directory:ssl}/nginx.crt
#################################
# Monitoring
#################################
[monitor-instance-parameter]
monitor-httpd-port = 9687
[monitor-conf-parameters]
private-path-list +=
$${directory:statistic}
$${directory:status}
# Add parts generated by template
[buildout]
extends =
${monitor-template:rendered}
parts =
dcron-service
nginx-service
nginx-listen-promise
activate-crontab-file
publish-global-rss
monitor-base
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/instance.cfg.in 0000664 0000000 0000000 00000002676 13712426147 0030614 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[dynamic-template-pullrdiffbackup]
recipe = slapos.recipe.template:jinja2
template = ${template-pullrdiffbackup:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-pullrdiffbackup.cfg
extensions = jinja2.ext.do
context =
key slave_instance_list instance-parameter:slave-instance-list
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:pullrdiffbackup}
# pullrdiffbackup = ${template-pullrdiffbackup:output}
pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
# [slap-parameter]
# slave-instance-list = []
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/software.cfg 0000664 0000000 0000000 00000005520 13712426147 0030224 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/rsync/buildout.cfg
../../component/openssh/buildout.cfg
../../component/grep/buildout.cfg
../../component/findutils/buildout.cfg
../../component/util-linux/buildout.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
parts =
rdiff-backup
dcron
logrotate
nginx
openssl
rsync
template
template-pullrdiffbackup
template-backup-script
template-crontab-line
slapos-cookbook
[rssgen-eggs]
recipe = zc.recipe.egg
interpreter = python-${:_buildout_section_name_}
eggs =
PyRSS2Gen
python-dateutil
##########################################################
# Service startup scripts and configuration files
##########################################################
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-dcron-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-backup-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-crontab-line]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-crontab]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[status2rss]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
[template-update-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/${:filename}
mode = 0644
##########################################################
# Buildout instance.cfg templates
##########################################################
[template-pullrdiffbackup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
rdiff-backup = 1.0.5+SlapOSPatched001
gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.3
PyRSS2Gen = 1.1
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/status2rss.py 0000664 0000000 0000000 00000002014 13712426147 0030413 0 ustar 00root root 0000000 0000000 import datetime
import uuid
import PyRSS2Gen
import sys
from dateutil.parser import parse
import base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
# ### Defaults
TITLE = sys.argv[1]
LINK = sys.argv[2]
DESCRIPTION = TITLE
items = []
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
time, statistic, desc = line.split(', ', 2)
rss_item = PyRSS2Gen.RSSItem(
title = desc,
description = "%s
" % "
".join(("%s, %s\nLastest statistic\n%s" % (time, desc,
open(statistic).read())).split("\n")),
pubDate = parse(time),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
)
items.append(rss_item)
### Build the rss feed
rss_feed = PyRSS2Gen.RSS2 (
title = TITLE,
link = LINK,
description = DESCRIPTION,
lastBuildDate = datetime.datetime.utcnow(),
items = items
)
print rss_feed.to_xml()
template-backup-script.sh.in 0000664 0000000 0000000 00000002764 13712426147 0033162 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver #!${dash-output:dash}
# don't run if rdiff-backup is still running
# in that case, the promise will detect that previous rdiff backup took too long and report an error
${util-linux:location}/bin/flock --nb $${:status_log}.lock true || exit
export HOME=$${directory:home}
# Inform about beginning of backup
${coreutils-output:echo} "`${coreutils-output:date} --iso-8601=seconds -u`, $${:statistic_log}, $${:hostname}, backup running" > $${:status_log}
${coreutils-output:echo} "Available only if backup succeed." > $${:statistic_log}
# set -e
cd $${:datadirectory}
${util-linux:location}/bin/flock $${:status_log}.lock \
${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \
--include='$${:include}' \
--exclude='**' \
--remote-schema '${openssh-output:ssh} -q -T -y -o "StrictHostKeyChecking no" -i $${:sshkey} -p $${:connection_port} %s $${:remote_schema}' \
$${:connection}::/ ./
RESULT=$?
# Inform about backup status
if [ $RESULT -eq 0 ]
then
${coreutils-output:echo} "`${coreutils-output:date} --iso-8601=seconds -u`, $${:statistic_log}, $${:hostname}, backup success" >> $${:status_log}
${findutils-output:find} rdiff-backup-data/ -maxdepth 1 -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average > $${:statistic_log}
else
${coreutils-output:echo} "`${coreutils-output:date} --iso-8601=seconds -u`, $${:statistic_log}, $${:hostname}, backup failed" >> $${:status_log}
fi
template-crontab-line.in 0000664 0000000 0000000 00000000072 13712426147 0032345 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver # $${:_buildout_section_name_}
$${:frequency} $${:script}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/template-crontab.in 0000664 0000000 0000000 00000000131 13712426147 0031473 0 ustar 00root root 0000000 0000000 # min(0-59) hours(0-23) day(1-31) month(1-12) dow(0-7) command
MAILTO=admins@erp5.org
template-dcron-service.sh.in 0000664 0000000 0000000 00000000463 13712426147 0033150 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver #!${dash-output:dash}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
exec ${dcron-output:crond} \
-s $${directory:cron-entries} \
-c $${directory:crontabs} \
-t $${directory:cronstamps} \
-f -l 5 \
-L $${dcron-service:logfile}
# -M cron_simplelogger
template-nginx-service.sh.in 0000664 0000000 0000000 00000001252 13712426147 0033163 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver #!${dash-output:dash}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
if [ ! -e $${nginx-configuration:ssl_crt} ]
then
${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
${openssl-output:openssl} req -new \
-subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
-key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
${openssl-output:openssl} x509 -req -days 365 \
-in $${nginx-configuration:ssl_csr} \
-signkey $${nginx-configuration:ssl_key} \
-out $${nginx-configuration:ssl_crt}
fi
exec ${nginx-output:nginx} \
-c $${nginx-configuration:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/template-nginx.cfg.in0000664 0000000 0000000 00000003332 13712426147 0031732 0 ustar 00root root 0000000 0000000 daemon off; # run in the foreground so supervisord can look after it
worker_processes 4;
pid $${directory:run}/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
error_log $${nginx-configuration:error_log};
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
default_type application/octet-stream;
include ${nginx-output:mime};
##
# Logging Settings
##
access_log $${nginx-configuration:access_log};
error_log $${nginx-configuration:error_log};
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
ssl on;
ssl_certificate $${nginx-configuration:ssl_crt};
ssl_certificate_key $${nginx-configuration:ssl_key};
fastcgi_temp_path $${directory:varnginx} 1 2;
uwsgi_temp_path $${directory:varnginx} 1 2;
scgi_temp_path $${directory:varnginx} 1 2;
client_body_temp_path $${directory:varnginx} 1 2;
proxy_temp_path $${directory:varnginx} 1 2;
## Only allow GET and HEAD request methods
if ($request_method !~ ^(GET|HEAD)$ ) {
return 444;
}
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location /
{
root $${directory:www};
# index index.html;
}
}
}
template-update-rss.sh.in 0000664 0000000 0000000 00000001250 13712426147 0032467 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver #!${dash-output:dash}
STATUS_DIR=$${directory:status}
RSS_DIR=$${directory:www}
TAIL=${coreutils-output:tail}
PYTHON=${buildout:bin-directory}/${rssgen-eggs:interpreter}
STATUS2RSS=${status2rss:output}
BASENAME=${coreutils-output:basename}
for status in $STATUS_DIR/*.txt
do
NAME=`$BASENAME $status`
$TAIL -n 1 -q $status | $PYTHON $STATUS2RSS "Backup status $NAME" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$NAME.rss" > $RSS_DIR/$NAME.rss
done
$TAIL -n 1 -q $STATUS_DIR/*.txt | $PYTHON $STATUS2RSS "Full backup status $${:global_rss}" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${:global_rss}" > $RSS_DIR/$${:global_rss}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/test/ 0000775 0000000 0000000 00000000000 13712426147 0026666 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/test/README.md 0000664 0000000 0000000 00000000050 13712426147 0030140 0 ustar 00root root 0000000 0000000 Tests for backupserver software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/test/setup.py 0000664 0000000 0000000 00000003676 13712426147 0030414 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.backupserver'
long_description = open("README.md").read()
setup(name=name,
version=version,
description="Test for SlapOS' backupserver",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/backupserver/test/test.py 0000664 0000000 0000000 00000004736 13712426147 0030231 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import httplib
import json
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestBackupServer(InstanceTestCase):
def test(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
# Check that there is a RSS feed
self.assertTrue('rss' in parameter_dict)
self.assertTrue(parameter_dict['rss'].startswith(
'https://[%s]:9443/' % (self._ipv6_address, )
))
result = requests.get(
parameter_dict['rss'], verify=False, allow_redirects=False)
# XXX crontab not triggered yet
self.assertEqual(
[httplib.NOT_FOUND, False],
[result.status_code, result.is_redirect]
)
# Check monitor
self.assertTrue('monitor-base-url' in parameter_dict)
self.assertTrue('monitor-setup-url' in parameter_dict)
result = requests.get(
parameter_dict['monitor-base-url'], verify=False, allow_redirects=False)
self.assertEqual(
[httplib.UNAUTHORIZED, False],
[result.status_code, result.is_redirect]
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/build-rina/ 0000775 0000000 0000000 00000000000 13712426147 0025241 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/build-rina/build.cfg 0000664 0000000 0000000 00000006536 13712426147 0027033 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/vm-img/debian.cfg
parts =
debuild-rina-tools
test-rina
download-cache = download-cache
[vm-run-base]
smp = 4
# estimate the of RAM needed depending on the number CPUs
mem = 2048
apt-sources =
apt-update =
for x in ${:apt-sources}
do echo deb [trusted=yes] file:`map $x` ./
done |sudo sh -c "cat >>/etc/apt/sources.list
apt-get update"
[debuild-rina-base]
<= vm-run-base
mount.rina = ${irati-stack:location}
cd-source =
git clone -snc core.sparseCheckout=true /mnt/rina rina
cd rina
echo /${:component} > .git/info/sparse-checkout
git reset --hard
cd ${:component}
# following lines are for debchange
export NAME=`git show -s --format=%cn`
export EMAIL=`git show -s --format=%ce`
dist=${:suite}
[ "$dist" ] && dist=`lsb_release -sc`/$dist || dist=UNRELEASED
prepare =
build-deps =
mk-build-deps -irs sudo -t 'apt-get -y'
build =
dpkg-buildpackage -uc -b -jauto
finalize =
df
cd ..
mv *.changes *.deb $PARTDIR
cd $PARTDIR
dpkg-scanpackages -m . /dev/null |gzip --best >Packages.gz
r=`apt-ftparchive release .`
echo "$r" > Release
command =
${:apt-update}
${:cd-source}
( ${:prepare}
${:build-deps}
${:build}
) 2>&1 |tee $PARTDIR/build.log
${:finalize}
[debuild-rina-kernel]
<= debuild-rina-base
component = linux
prepare =
debian/dch-snapshot --force-distribution -D $dist
sudo apt-get -y install kernel-wedge quilt python-six
QUILT_PATCHES=$PWD/debian/patches QUILT_PC=.pc quilt push --quiltrc - -a -q --fuzz=0
debian/rules debian/rules.gen || :
build =
arch=`dpkg-architecture -qDEB_HOST_ARCH`
fakeroot make -j${:smp} -f debian/rules.gen binary-arch_$${arch}_none binary-libc-dev_$${arch}
dpkg-genchanges -b -UBinary -UDescription \
>../linux_`dpkg-parsechangelog -S Version`_$${arch}.changes
[debuild-librina]
<= debuild-rina-base
component = librina
apt-sources = ${debuild-rina-kernel:location}
prepare =
debian/rules DIST=$dist
[debuild-rinad]
<= debuild-librina
component = rinad
apt-sources = ${debuild-librina:location}
[rina-tools-deps]
apt-sources = ${debuild-librina:location} ${debuild-rinad:location}
[debuild-rina-tools]
<= debuild-librina
rina-tools-deps
component = rina-tools
# check that the built packages install
finalize =
sudo dpkg -i ../*.deb
${debuild-librina:finalize}
[test-rina]
<= vm-run-base
rina-tools-deps
mount.download-cache = ${buildout:download-cache}
mount.rina = ${irati-stack:location}
mount.slapos = ${:_profile_base_location_}/../..
mount.slapos.package = ${slapos.package-repository:location}
commands = install test
install =
${:apt-update}
sudo apt-get -y install librinad-dev rinad
mkdir software
cd software
cat <buildout.cfg
[buildout]
extends = /mnt/slapos/software/hellorina/software.cfg
develop = /mnt/slapos
download-cache = /mnt/download-cache
[slapos-cookbook-develop]
recipe =
setup =
[irati-stack]
repository = /mnt/rina
shared = true
[versions]
slapos.cookbook =
EOF
python -S /mnt/buildout/bin/buildout bootstrap
MAKEFLAGS=-j${:smp} bin/buildout
arch=`dpkg-architecture -qDEB_HOST_ARCH`
sudo sh -c "/mnt/slapos.package/playbook/roles/rina/gen-ipcm-conf
systemctl enable ipcm-re6st
dpkg -i `map ${debuild-rina-kernel:location}`/linux-image-*-$${arch}_*.deb"
reboot
test = ( set -x
# TODO: more tests
grep re6st/1// /sys/rina/ipcps/1/name
) 2>&1 |tee $PARTDIR/result
instance-build-rina-input-schema.json 0000664 0000000 0000000 00000001322 13712426147 0034276 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/build-rina {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to build/publish RINA packages and test hellorina SR.",
"type": "object",
"additionalProperties": false,
"properties": {
"publish": {
"description": "Upload built packages automatically to a Debian repository when successful.",
"type": "object",
"required": [
"suite",
"host",
"key"
],
"properties": {
"suite": {
"type": "string"
},
"host": {
"type": "string"
},
"port": {
"type": "integer",
"default": 22
},
"key": {
"type": "string"
}
}
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/build-rina/runTestSuite.in 0000664 0000000 0000000 00000013747 13712426147 0030263 0 ustar 00root root 0000000 0000000 #!{{parameter_dict['runTestSuite_py']}}
from __future__ import print_function
import argparse, os, subprocess, sys, traceback
from time import gmtime, strftime, time
# These are the 2 modules to reuse when using ERP5 for managing test bots.
# What we do here is currently too new to reuse more from testsuite.
from erp5.util import taskdistribution
from erp5.util.testsuite import format_command
from zc.buildout.buildout import Buildout
{% set vm = parameter_dict['vm'] -%}
dist_list = {{vm['dists'].split()}}
publish = {{slapparameter_dict.get('publish')}}
# ERP5 must be changed to only distinguish SKIP/EXPECTED/UNEXPECTED,
# instead of SKIP/FAIL/ERROR. Unlike NEO, we categorize XFAIL as SKIP
# so that the overall status is PASS if there's no FAIL/XPASS/ERROR.
STAT_MAP = dict(
TOTAL = 'test_count',
PASS = None,
SKIP = 'skip_count',
XFAIL = 'skip_count',
FAIL = 'failure_count',
XPASS = 'failure_count',
ERROR = 'error_count',
)
class DummyTestResult:
class DummyTestResultLine:
def stop(self, duration, stdout='', **kw):
print('\n' + stdout)
print('Ran in %.3fs' % duration)
done = 0
def __init__(self, test_name_list):
self.test_name_list = test_name_list
def start(self):
test_result_line = self.DummyTestResultLine()
try:
test_result_line.name = self.test_name_list[self.done]
except IndexError:
return
self.done += 1
return test_result_line
def main():
os.environ.update({k: v.strip() % os.environ
for k, v in {{parameter_dict['environment'].items()}}})
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', type=int,
help='Number of CPUs to use for the VM')
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
args = parser.parse_args()
test_title = args.test_suite_title or args.test_suite
if args.master_url:
tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision,
dist_list,
args.test_node_title,
test_title=test_title,
project_title=args.project_title)
if test_result is None:
return
else:
test_result = DummyTestResult(dist_list)
Buildout._setup_logging = lambda self: None
fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666)
try:
os.write(fd, """\
[buildout]
offline = true
relative-paths = true
""")
Buildout('buildout.cfg', {}).bootstrap(None)
os.write(fd, """\
extends = {{parameter_dict['profile_base_location']}}/build.cfg
develop-eggs-directory = {{buildout['develop-eggs-directory']}}
eggs-directory = {{buildout['eggs-directory']}}
download-cache = {{parameter_dict['download-cache']}}
[vm-run-base]
environment =
vm = {{vm['location']}}
[irati-stack]
location = {{parameter_dict['irati_stack']}}
[slapos.package-repository]
location = {{parameter_dict['slapos_package']}}
""")
finally:
os.close(fd)
librina_log = os.path.join('parts', 'debuild-librina', 'build.log')
stderr_write = sys.stderr.write
while 1:
test_result_line = test_result.start()
if not test_result_line:
break
dist = test_result_line.name
cmd = [os.path.join('bin', 'buildout'),
'vm-run-base:dist=' + dist,
'debuild-rina-base:suite=' + (publish['suite'] if publish else ''),
]
if args.node_quantity:
cmd.append('vm-run-base:smp=%s' % args.node_quantity)
status_dict = {'command': format_command(*cmd)}
print('$', status_dict['command'])
# Wanted on test result lines:
# status: UNKNOWN in case of buildout failure
# (even if the test suite could be run)
# output: test suite summary if any
# error: buildout traceback or test suite log
start = time()
try:
try:
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
stderr = []
while 1:
line = p.stderr.readline()
if not line:
break
stderr_write(line)
stderr.append(line)
returncode = p.wait()
finally:
end = time()
del p
if returncode:
iter_err = enumerate(reversed(stderr), 1)
for i, line in iter_err:
if line == "Traceback (most recent call last):\n":
for i, line in iter_err:
if line == '\n':
break
for i, line in iter_err:
if line[0] != ' ':
break
break
if line == "While:\n":
del stderr[:-i]
status_dict['stderr'] = ''.join(stderr)
with open(librina_log) as f:
log = f.readlines()
del log[:log.index('make check-TESTS\n')]
for i, line in enumerate(log):
if line.startswith('Testsuite summary'):
del log[log.index(log[i+1], i+2):]
status_dict['stdout'] = ''.join(log[i:])
stat = {}
for line in log[i+2:]:
k, v = line[2:].split(':')
k = STAT_MAP[k]
if k:
stat[k] = stat.get(k, 0) + int(v.strip())
if not returncode:
status_dict.update(stat)
status_dict.setdefault('stderr', ''.join(log[:i-1]))
break
except Exception:
status_dict.setdefault('stderr', traceback.format_exc())
test_result_line.stop(
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration = end - start,
**status_dict)
# TODO: upload packages if 'publish' parameter is given
if __name__ == "__main__":
main()
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/build-rina/software.cfg 0000664 0000000 0000000 00000005551 13712426147 0027562 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../component/vm-img/debian.cfg
../../component/rina-tools/buildout.cfg
parts =
slapos-cookbook
template
download-cache = ${:directory}/download-cache
[template]
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = inline:
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
parts = runTestSuite
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
mode = 0755
context =
{##} section buildout buildout
{##} section parameter_dict runTestSuite-parameters
{##} key slapparameter_dict slap-configuration:configuration
[runTestSuite-parameters]
environment = {{dumps(environment)}}
vm = {{dumps(vm)}}
download-cache = ${buildout:download-cache}
runTestSuite_py = ${buildout:bin-directory}/${runTestSuite_py:interpreter}
profile_base_location = ${:_profile_base_location_}
irati_stack = ${irati-stack:location}
slapos_package = ${slapos.package-repository:location}
context =
section environment vm-run-environment
section vm vm-debian
[runTestSuite_py]
recipe = zc.recipe.egg
eggs = erp5.util
zc.buildout
interpreter = ${:_buildout_section_name_}
[irati-stack]
sparse-checkout = /.gitignore
# When run from erp5.util:testnode (and this always the case), shared=true is
# added to this section and the path in .git/objects/info/alternates is outside
# the 'rina' mount point of the VM. Let's mount the testnode working copy
# directly. Comment the following 2 lines if 'repository' is a URL.
recipe =
location = ${:repository}
[slapos.package-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.package.git
git-executable = ${git:location}/bin/git
sparse-checkout = /playbook/roles/rina
[vm-debian]
# building a generic Debian kernel uses a lot of space
size = 16Gi
dists = debian-jessie
packages +=
# generic (another SR that build packages automatically would use the same list)
apt-utils build-essential devscripts equivs lsb-release
# specific
git ca-certificates python
# biggest and common build-deps for RINA
dh-autoreconf pkg-config doxygen maven xmlto
# hellorina (shouldn't parts like lxml-python depend on the python of the SR?)
python-dev
[versions]
slapos.recipe.template = 4.4
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/buildout-testing/ 0000775 0000000 0000000 00000000000 13712426147 0026515 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/buildout-testing/runTestSuite.in 0000664 0000000 0000000 00000007650 13712426147 0031533 0 ustar 00root root 0000000 0000000 #!{{runTestSuite_py}}
from __future__ import print_function
import argparse, os, re, subprocess, sys
from time import gmtime, strftime, time
from erp5.util import taskdistribution
from erp5.util.testsuite import SubprocessError, TestSuite
from pkg_resources import get_distribution
from zc.buildout.buildout import Buildout
if str is bytes:
str2bytes = lambda s: s
else:
str2bytes = lambda s: s.encode()
slapos_buildout = {{repr(slapos_buildout)}}
test_dict = {
'zc.buildout': slapos_buildout,
'zc.recipe.egg': os.path.join(slapos_buildout, 'zc.recipe.egg_'),
}
class DummyTestResult:
class DummyTestResultLine:
def stop(self, **kw):
pass
done = 0
def __init__(self, test_name_list):
self.test_name_list = test_name_list
def start(self):
test_result_line = self.DummyTestResultLine()
try:
test_result_line.name = self.test_name_list[self.done]
except IndexError:
return
self.done += 1
return test_result_line
class BuildoutTestSuite(TestSuite):
RUN_RE = re.compile(
br'Ran (?P\d+) tests with'
br' (?P\d+) failures,'
br' (?P\d+) errors and'
br' (?P\d+) skipped in')
def run(self, test):
start = time()
try:
status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'),
'--test-path', os.path.join(test_dict[test], 'src'))
except SubprocessError as e:
status_dict = e.status_dict
end = time()
status_dict.update(
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration = end - start)
search = self.RUN_RE.search(status_dict['stdout'])
if search:
groupdict = search.groupdict()
status_dict.update(
test_count = int(groupdict['all_tests']),
error_count = int(groupdict['errors']),
failure_count = int(groupdict['failures']),
skip_count = int(groupdict['skips']))
return status_dict
def main():
os.environ['TEMP'] = {{repr(temp_directory)}}
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
args = parser.parse_args()
test_title = args.test_suite_title or args.test_suite
if args.master_url:
tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision,
list(test_dict),
args.test_node_title,
test_title=test_title,
project_title=args.project_title)
if test_result is None:
return
else:
test_result = DummyTestResult(list(test_dict))
fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
try:
os.write(fd, str2bytes("""\
[buildout]
develop =%s
parts = testrunner
newest = false
versions = versions
[versions]
%s
zope.exceptions = 4.3
zope.interface = 4.7.2
zope.testing = 4.7
[testrunner]
recipe = zc.recipe.egg
eggs =
${:recipe}
zope.testing
zope.testrunner
scripts =
zope-testrunner
extra-paths =
%s
""" % (''.join('\n ' + x for x in test_dict.values()),
'\n'.join(x + ' =' for x in test_dict),
get_distribution('manuel').location,
)))
finally:
os.close(fd)
Buildout('buildout.cfg', {}).install(None)
test_suite = BuildoutTestSuite(1)
while 1:
test_result_line = test_result.start()
if not test_result_line:
break
test_result_line.stop(**test_suite.run(test_result_line.name))
if __name__ == "__main__":
main()
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/buildout-testing/software-py3.cfg 0000664 0000000 0000000 00000000075 13712426147 0031543 0 ustar 00root root 0000000 0000000 [buildout]
extends =
software.cfg
[python]
part = python3
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/buildout-testing/software.cfg 0000664 0000000 0000000 00000003517 13712426147 0031036 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/python-manuel/buildout.cfg
../../stack/slapos.cfg
parts =
slapos-cookbook
template
[slapos.buildout-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.buildout.git
git-executable = ${git:location}/bin/git
[runTestSuite_py]
recipe = zc.recipe.egg
eggs = erp5.util
zc.buildout
${manuel:egg}
scripts = ${:interpreter}
interpreter = ${:_buildout_section_name_}
[template]
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template =
inline:
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
parts = runTestSuite
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[slapos.buildout-repository]
recipe = slapos.recipe.build:gitclone
repository = ${slapos.buildout-repository:location}
git-executable = ${git:location}/bin/git
shared = true
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
mode = 0755
context =
key slapparameter_dict slap-configuration:configuration
key slapos_buildout slapos.buildout-repository:location
key temp_directory directory:tmp
raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter}
[versions]
slapos.recipe.template = 4.4
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/ 0000775 0000000 0000000 00000000000 13712426147 0026114 5 ustar 00root root 0000000 0000000 CHANGES.caddy_frontend.rst 0000664 0000000 0000000 00000003032 13712426147 0032617 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend Changes
=======
Here are listed the most important changes, which might affect upgrades.
1.0.159 (2020-07-30)
--------------------
* logs are ensured to be available in slave's ``log-access-url``
* logs from backend Haproxy are also available to slaves
1.0.158 (2020-07-24)
--------------------
* manual customisation of profiles has been dropped, as not used, dropped keys are ``apache_custom_http``, ``apache_custom_https``, ``caddy_custom_http``, ``caddy_custom_https`` from slaves and ``-frontend-authorized-slave-string`` from master
* ``re6st-optimal-test`` has been dropped from slave
* QUIC is dropped, as was not used and has been superseded by HTTP/3, dropped key is ``enable-quic`` from master
* haproxy is used as a gateway to backends:
* ``automatic-internal-backend-client-caucase-csr`` switch for master is introduced to control it CSR signing
* ``proxy-try-duration`` and ``proxy-try-interval`` has been dropped, as Caddy is not used anymore to connect to the backend, and instead ``backend-connect-timeout`` and ``backend-connect-retries`` is used, as it comes from Haproxy
* ``backend-client-caucase-url`` is returned in master and slave, so that backends can use caucase to fetch CA from frontend cluster
* ``request-timeout`` is supported per slave, as now it became possible
* ``authenticate-to-backend`` is added for master and slave, defaulting to False, to have control over cluster default authentication, and make it possible to do it per slave
1.0.149 (2020-05-05)
--------------------
* no changes noted
README.caddy_frontend.rst 0000664 0000000 0000000 00000051336 13712426147 0032516 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend ==============
Caddy Frontend
==============
Frontend system using Caddy, based on apache-frontend software release, allowing to rewrite and proxy URLs like myinstance.myfrontenddomainname.com to real IP/URL of myinstance.
Caddy Frontend works using the master instance / slave instance design. It means that a single main instance of Caddy will be used to act as frontend for many slaves.
Software type
=============
Caddy frontend is available in 4 software types:
* ``default`` : The standard way to use the Caddy frontend configuring everything with a few given parameters
* ``custom-personal`` : This software type allow each slave to edit its Caddy configuration file
* ``default-slave`` : XXX
* ``custom-personal-slave`` : XXX
About frontend replication
==========================
Slaves of the root instance are sent as a parameter to requested frontends which will process them. The only difference is that they will then return the would-be published information to the root instance instead of publishing it. The root instance will then do a synthesis and publish the information to its slaves. The replicate instance only use 5 type of parameters for itself and will transmit the rest to requested frontends.
These parameters are:
* ``-frontend-type`` : the type to deploy frontends with. (default to 2)
* ``-frontend-quantity`` : The quantity of frontends to request (default to "default")
* ``-frontend-i-state``: The state of frontend i
* ``-frontend-config-i-foo``: Frontend i will be requested with parameter foo
* ``-frontend-software-release-url``: Software release to be used for frontends, default to the current software release
* ``-sla-i-foo`` : where "i" is the number of the concerned frontend (between 1 and "-frontend-quantity") and "foo" a sla parameter.
For example::
3
custom-personal
stopped
COMP-1234
https://lab.nexedi.com/nexedi/slapos/raw/someid/software/caddy-frontend/software.cfg
will request the third frontend on COMP-1234. All frontends will be of software type ``custom-personal``. The second frontend will be requested with the state stopped
*Note*: the way slaves are transformed to a parameter avoid modifying more than 3 lines in the frontend logic.
**Important NOTE**: The way you ask for slave to a replicate frontend is the same as the one you would use for the software given in "-frontend-quantity". Do not forget to use "replicate" for software type. XXXXX So far it is not possible to do a simple request on a replicate frontend if you do not know the software_guid or other sla-parameter of the master instance. In fact we do not know yet the software type of the "requested" frontends. TO BE IMPLEMENTED
XXX Should be moved to specific JSON File
Extra-parameter per frontend with default::
ram-cache-size = 1G
disk-cache-size = 8G
How to deploy a frontend server
===============================
This is to deploy an entire frontend server with a public IPv4. If you want to use an already deployed frontend to make your service available via ipv4, switch to the "Example" parts.
First, you will need to request a "master" instance of Caddy Frontend with:
* A ``domain`` parameter where the frontend will be available
* A ``public-ipv4`` parameter to state which public IPv4 will be used
like::
moulefrite.org
xxx.xxx.xxx.xxx
Then, it is possible to request many slave instances (currently only from slapconsole, UI doesn't work yet) of Caddy Frontend, like::
instance = request(
software_release=caddy_frontend,
partition_reference='frontend2',
shared=True,
partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
)
Those slave instances will be redirected to the "master" instance, and you will see on the "master" instance the associated proper directives of all slave instances.
Finally, the slave instance will be accessible from: https://someidentifier.moulefrite.org.
About SSL and SlapOS Master Zero Knowledge
==========================================
**IMPORTANT**: One Caddy can not serve more than one specific SSL site and be compatible with obsolete browser (i.e.: IE8). See http://wiki.apache.org/httpd/NameBasedSSLVHostsWithSNI
SSL keys and certificates are directly send to the frontend cluster in order to follow zero knowledge principle of SlapOS Master.
*Note*: Until master partition or slave specific certificate is uploaded each slave is served with fallback certificate. This fallback certificate is self signed, does not match served hostname and results with lack of response on HTTPs.
Obtaining CA for KeDiFa
-----------------------
KeDiFa uses caucase and so it is required to obtain caucase CA certificate used to sign KeDiFa SSL certificate, in order to be sure that certificates are sent to valid KeDiFa.
The easiest way to do so is to use caucase.
On some secure and trusted box which will be used to upload certificate to master or slave frontend partition install caucase https://pypi.org/project/caucase/
Master and slave partition will return key ``kedifa-caucase-url``, so then create and start a ``caucase-updater`` service::
caucase-updater \
--ca-url "${kedifa-caucase-url}" \
--cas-ca "${frontend_name}.caucased.ca.crt" \
--ca "${frontend_name}.ca.crt" \
--crl "${frontend_name}.crl"
where ``frontend_name`` is a frontend cluster to which you will upload the certificate (it can be just one slave).
Make sure it is automatically started when trusted machine reboots: you want to have it running so you can forget about it. It will keep KeDiFa's CA certificate up to date when it gets renewed so you know you are still talking to the same service as when you previously uploaded the certificate, up to the original upload.
Master partition
----------------
After requesting master partition it will return ``master-key-generate-auth-url`` and ``master-key-upload-url``.
Doing HTTP GET on ``master-key-generate-auth-url`` will return authentication token, which is used to communicate with ``master-key-upload-url``. This token shall be stored securely.
By doing HTTP PUT to ``master-key-upload-url`` with appended authentication token it is possible to upload PEM bundle of certificate, key and any accompanying CA certificates to the master.
Example sessions is::
request(...)
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" master-key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem master-key-upload-url+authtoken
This replaces old request parameters:
* ``apache-certificate``
* ``apache-key``
* ``apache-ca-certificate``
(*Note*: They are still supported for backward compatibility, but any value send to the ``master-key-upload-url`` will supersede information from SlapOS Master.)
Slave partition
---------------
After requesting slave partition it will return ``key-generate-auth-url`` and ``key-upload-url``.
Doing HTTP GET on ``key-generate-auth-url`` will return authentication token, which is used to communicate with ``key-upload-url``. This token shall be stored securely.
By doing HTTP PUT to ``key-upload-url`` with appended authentication token it is possible to upload PEM bundle of certificate, key and any accompanying CA certificates to the master.
Example sessions is::
request(...)
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem key-upload-url+authtoken
This replaces old request parameters:
* ``ssl_crt``
* ``ssl_key``
* ``ssl_ca_crt``
(*Note*: They are still supported for backward compatibility, but any value send to the ``key-upload-url`` will supersede information from SlapOS Master.)
Instance Parameters
===================
Master Instance Parameters
--------------------------
The parameters for instances are described at `instance-caddy-input-schema.json `_.
Here some additional informations about the parameters listed, below:
domain
~~~~~~
Name of the domain to be used (example: mydomain.com). Sub domains of this domain will be used for the slave instances (example: instance12345.mydomain.com). It is then recommended to add a wild card in DNS for the sub domains of the chosen domain like::
*.mydomain.com. IN A 123.123.123.123
Using the IP given by the Master Instance. "domain" is a mandatory Parameter.
public-ipv4
~~~~~~~~~~~
Public ipv4 of the frontend (the one Caddy will be indirectly listening to)
port
~~~~
Port used by Caddy. Optional parameter, defaults to 4443.
plain_http_port
~~~~~~~~~~~~~~~
Port used by Caddy to serve plain http (only used to redirect to https).
Optional parameter, defaults to 8080.
Slave Instance Parameters
-------------------------
The parameters for instances are described at `instance-slave-caddy-input-schema.json `_.
Here some additional informations about the parameters listed, below:
path
~~~~
Only used if type is "zope".
Will append the specified path to the "VirtualHostRoot" of the zope's VirtualHostMonster.
"path" is an optional parameter, ignored if not specified.
Example of value: "/erp5/web_site_module/hosting/"
url
~~~
Necessary to activate cache. ``url`` of backend to use.
``url`` is an optional parameter.
Example: http://mybackend.com/myresource
domain
~~~~~~
Necessary to activate cache.
The frontend will be accessible from this domain.
``domain`` is an optional parameter.
Example: www.mycustomdomain.com
enable_cache
~~~~~~~~~~~~
Necessary to activate cache.
``enable_cache`` is an optional parameter.
Examples
========
Here are some example of how to make your SlapOS service available through an already deployed frontend.
Simple Example (default)
------------------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
redirected and accessible from the proxy::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
}
)
Zope Example (default)
----------------------
Request slave frontend instance using a Zope backend so that
https://[1:2:3:4:5:6:7:8]:1234 will be redirected and accessible from the
proxy::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"type":"zope",
}
)
Advanced example
-----------------
Request slave frontend instance using a Zope backend, with Varnish activated,
listening to a custom domain and redirecting to /erp5/ so that
https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
the proxy::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"enable_cache":"true",
"type":"zope",
"path":"/erp5",
"domain":"mycustomdomain.com",
}
)
Simple Example
---------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
Simple Cache Example - XXX - to be written
------------------------------------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"domain": "www.example.org",
"enable_cache": "True",
Advanced example - XXX - to be written
--------------------------------------
Request slave frontend instance using custom apache configuration, willing to use cache and ssl certificates.
Listening to a custom domain and redirecting to /erp5/ so that
https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
the proxy::
instance = request(
software_release=caddy_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
software_type="custom-personal",
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"enable_cache":"true",
"type":"zope",
"path":"/erp5",
"domain":"example.org",
"ssl_key":"-----BEGIN RSA PRIVATE KEY-----
XXXXXXX..........XXXXXXXXXXXXXXX
-----END RSA PRIVATE KEY-----",
"ssl_crt":'-----BEGIN CERTIFICATE-----
XXXXXXXXXXX.............XXXXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----',
"ssl_ca_crt":'-----BEGIN CERTIFICATE-----
XXXXXXXXX...........XXXXXXXXXXXXXXXXX
-----END CERTIFICATE-----',
"ssl_csr":'-----BEGIN CERTIFICATE REQUEST-----
XXXXXXXXXXXXXXX.............XXXXXXXXXXXXXXXXXX
-----END CERTIFICATE REQUEST-----',
}
)
Promises
========
Note that in some cases promises will fail:
* not possible to request frontend slave for monitoring (monitoring frontend promise)
* no slaves present (configuration promise and others)
* no cached slave present (configuration promise and others)
This is known issue and shall be tackled soon.
KeDiFa
======
Additional partition with KeDiFa (Key Distribution Facility) is by default requested on the same computer as master frontend partition.
By adding to the request keys like ``-sla-kedifa-`` it is possible to provide SLA information for kedifa partition. Eg to put it on computer ``couscous`` it shall be ``-sla-kedifa-computer_guid: couscous``.
Notes
=====
It is not possible with slapos to listen to port <= 1024, because process are
not run as root.
Solution 1 (iptables)
---------------------
It is a good idea then to go on the node where the instance is
and set some ``iptables`` rules like (if using default ports)::
iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 443 -j DNAT --to-destination {listening_ipv4}:4443
iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 80 -j DNAT --to-destination {listening_ipv4}:8080
ip6tables -t nat -A PREROUTING -p tcp -d {public_ipv6} --dport 443 -j DNAT --to-destination {listening_ipv6}:4443
ip6tables -t nat -A PREROUTING -p tcp -d {public_ipv6} --dport 80 -j DNAT --to-destination {listening_ipv6}:8080
Where ``{public_ipv[46]}`` is the public IP of your server, or at least the LAN IP to where your NAT will forward to, and ``{listening_ipv[46]}`` is the private ipv4 (like 10.0.34.123) that the instance is using and sending as connection parameter.
Additionally in order to access the server by itself such entries are needed in ``OUTPUT`` chain (as the internal packets won't appear in the ``PREROUTING`` chain)::
iptables -t nat -A OUTPUT -p tcp -d {public_ipv4} --dport 443 -j DNAT --to {listening_ipv4}:4443
iptables -t nat -A OUTPUT -p tcp -d {public_ipv4} --dport 80 -j DNAT --to {listening_ipv4}:8080
ip6tables -t nat -A OUTPUT -p tcp -d {public_ipv6} --dport 443 -j DNAT --to {listening_ipv6}:4443
ip6tables -t nat -A OUTPUT -p tcp -d {public_ipv6} --dport 80 -j DNAT --to {listening_ipv6}:8080
Solution 2 (network capability)
-------------------------------
It is also possible to directly allow the service to listen on 80 and 443 ports using the following command::
setcap 'cap_net_bind_service=+ep' /opt/slapgrid/$CADDY_FRONTEND_SOFTWARE_RELEASE_MD5/go.work/bin/caddy
setcap 'cap_net_bind_service=+ep' /opt/slapgrid/$CADDY_FRONTEND_SOFTWARE_RELEASE_MD5/parts/6tunnel/bin/6tunnel
Then specify in the master instance parameters:
* set ``port`` to ``443``
* set ``plain_http_port`` to ``80``
Authentication to the backend
=============================
The cluster generates CA served by caucase, available with ``backend-client-caucase-url`` return parameter.
Then, each slave configured with ``authenticate-to-backend`` to true, will use a certificate signed by this CA while accessing https backend.
This allows backends to:
* restrict access only from some frontend clusters
* trust values (like ``X-Forwarded-For``) sent by the frontend
Technical notes
===============
Instantiated cluster structure
------------------------------
Instantiating caddy-frontend results with a cluster in various partitions:
* master (the controlling one)
* kedifa (contains kedifa server)
* caddy-frontend-N which contains the running processes to serve sites - this partition can be replicated by ``-frontend-quantity`` parameter
It means sites are served in ``caddy-frontend-N`` partition, and this partition is structured as:
* Caddy serving the browser [client-facing-caddy]
* (optional) Apache Traffic Server for caching [ats]
* Haproxy as a way to communicate to the backend [backend-facing-haproxy]
* some other additional tools (6tunnel, monitor, etc)
In case of slaves without cache (``enable_cache = False``) the request will travel as follows::
client-facing-caddy --> backend-facing-haproxy --> backend
In case of slaves using cache (``enable_cache = True``) the request will travel as follows::
client-facing-caddy --> ats --> backend-facing-haproxy --> backend
Usage of Haproxy as a relay to the backend allows much better control of the backend, removes the hassle of checking the backend from Caddy and allows future developments like client SSL certificates to the backend or even health checks.
Kedifa implementation
---------------------
`Kedifa `_ server runs on kedifa partition.
Each `caddy-frontend-N` partition downloads certificates from the kedifa server.
Caucase (exposed by ``kedifa-caucase-url`` in master partition parameters) is used to handle certificates for authentication to kedifa server.
If ``automatic-internal-kedifa-caucase-csr`` is enabled (by default it is) there are scripts running on master partition to simulate human to sign certificates for each caddy-frontend-N node.
Support for X-Real-Ip and X-Forwarded-For
-----------------------------------------
X-Forwarded-For and X-Real-Ip are transmitted to the backend, but only for IPv4 access to the frontend. In case of IPv6 access, the provided IP will be wrong, because of using 6tunnel.
Automatic Internal Caucase CSR
------------------------------
Cluster is composed on many instances, which are landing on separate partitions, so some way is needed to bootstrap trust between the partitions.
There are two ways to achieve it:
* use default, Automatic Internal Caucase CSR used to replace human to sign CSRs against internal CAUCASEs automatic bootstrap, which leads to some issues, described later
* switch to manual bootstrap, which requires human to create and manage user certificate (with caucase-updater) and then sign new frontend nodes appearing in the system
The issues during automatic bootstrap are:
* rouge or hacked SlapOS Master can result with adding rouge frontend nodes to the cluster, which will be trusted, so it will be possible to fetch all certificates and keys from Kedifa or to login to backends
* when new node is added there is short window, when rouge person is able to trick automatic signing, and have it's own node added
In both cases promises will fail on node which is not able to get signed, but in case of Kedifa the damage already happened (certificates and keys are compromised). So in case if cluster administrator wants to stay on the safe side, both automatic bootstraps shall be turned off.
How the automatic signing works
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Having in mind such structure:
* instance with caucase: ``caucase-instance``
* N instances which want to get their CSR signed: ``csr-instance``
In ``caucase-instance`` CAUCASE user is created by automatically signing one user certificate, which allows to sign service certificates.
The ``csr-instance`` creates CSR, extracts the ID of the CSR, exposes it via HTTP and ask caucase on ``caucase-instance`` to sign it. The ``caucase-instance`` checks that exposed CSR id matches the one send to caucase and by using created user to signs it.
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/buildout.hash.cfg 0000664 0000000 0000000 00000010042 13712426147 0031343 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg.in
md5sum = dae3bf6daf851b5610a1a6bd83057b29
[template-common]
filename = instance-common.cfg.in
md5sum = 5784bea3bd608913769ff9a8afcccb68
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 43b3763d4086283e9c7c886e3b89e54b
[template-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 19debfbc27c464f451b1eb5bb5ce3c84
[template-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 8f518a7b543126ce91fe218255fb201c
[template-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
md5sum = 7e3ee70c447f8203273d78f66ab519c3
[template-caddy-frontend-configuration]
_update_hash_filename_ = templates/Caddyfile.in
md5sum = 2503056e35463e045db3329bb8b6fae8
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
md5sum = 0979a03476e86bf038516c9565dadc17
[template-not-found-html]
_update_hash_filename_ = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost]
_update_hash_filename_ = templates/default-virtualhost.conf.in
md5sum = 266f175dbdfc588af7a86b0b1884fe73
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = 68a7758ca8f8b544ba9bc756824be3d3
[template-log-access]
_update_hash_filename_ = templates/template-log-access.conf.in
md5sum = f8068179333ce19e95df561c70073857
[template-empty]
_update_hash_filename_ = templates/empty.in
md5sum = 7155b18edfe128825b8d1f48071454a6
[template-wrapper]
_update_hash_filename_ = templates/wrapper.in
md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = f3f31188de56bb35383335b3219537f4
[template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2
md5sum = baf7b89cc9ab5506100b0c900808c1ea
[template-trafficserver-logging-config]
_update_hash_filename_ = templates/trafficserver/logging.config.jinja2
md5sum = 6aed31174dc262ced02f31624321df41
[template-nginx-eventsource-slave-virtualhost]
_update_hash_filename_ = templates/nginx-eventsource-slave.conf.in
md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-caddy-lazy-script-call]
_update_hash_filename_ = templates/apache-lazy-script-call.sh.in
md5sum = 77d60840591de67b64ab3572e46273a0
[template-graceful-script]
_update_hash_filename_ = templates/graceful-script.sh.in
md5sum = 061cc244558fd3af2b6bacf17cae5555
[template-validate-script]
_update_hash_filename_ = templates/validate-script.sh.in
md5sum = 53e5d7ba2827bff003051f74f24ffe4f
[template-configuration-state-script]
_update_hash_filename_ = templates/configuration-state-script.sh.in
md5sum = 4d2537d2698d32a7e909989f8778d144
[template-rotate-script]
_update_hash_filename_ = templates/rotate-script.sh.in
md5sum = 8c150e1e6c993708d31936742f3a7302
[caddyprofiledeps-setup]
filename = setup.py
md5sum = d9b6476bb0b36cf463fddb00d41dfbaa
[caddyprofiledeps-dummy]
filename = caddyprofiledummy.py
md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
filename = instance-kedifa.cfg.in
md5sum = f77744e52f9d028c39f99cfbf31eadaf
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
md5sum = be899b04e1aa652ed510f20d4ea523dd
caddyprofiledummy.py 0000664 0000000 0000000 00000000231 13712426147 0032124 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend class Recipe(object):
def __init__(self, *args, **kwargs):
pass
def install(self):
return []
def update(self):
return self.install()
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/common.cfg 0000664 0000000 0000000 00000015404 13712426147 0030071 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/caddy/buildout.cfg
../../component/gzip/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/trafficserver/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/haproxy/buildout.cfg
../../stack/caucase/buildout.cfg
# Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg
parts +=
caucase-eggs
template
template-caddy-frontend
template-caddy-replicate
caddy
logrotate
rdiff-backup
caddyprofiledeps
kedifa-develop
kedifa
[kedifa-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = d6bbd7db215e12871c1536f22a8fbf994227270c
[kedifa-develop]
recipe = zc.recipe.egg:develop
setup = ${kedifa-repository:location}
[kedifa]
recipe = zc.recipe.egg
eggs =
${python-cryptography:egg}
kedifa
[caddyprofiledeps-setup]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/setup.py
[caddyprofiledeps-dummy]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/caddyprofiledummy.py
[caddyprofiledeps-prepare]
recipe = plone.recipe.command
stop-on-error = True
location = ${buildout:parts-directory}/${:_buildout_section_name_}
update-command = ${:command}
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
cp ${caddyprofiledeps-setup:target} ${:location}/ &&
cp ${caddyprofiledeps-dummy:target} ${:location}/
[caddyprofiledeps-develop]
recipe = zc.recipe.egg:develop
setup = ${caddyprofiledeps-prepare:location}
[caddyprofiledeps]
depends = ${caddyprofiledeps-develop:recipe}
recipe = zc.recipe.egg
eggs =
caddyprofiledeps
websockify
collective.recipe.shelloutput
[template-common]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-common.cfg.in
rendered = ${buildout:directory}/instance-common.cfg
mode = 0644
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
[template-frontend-parameter-section]
common_profile = ${template-common:rendered}
logrotate_base_instance = ${template-logrotate-base:rendered}
bin_directory = ${buildout:bin-directory}
sixtunnel = ${6tunnel:location}
caddy = ${caddy:output}
caddy_location = ${caddy:location}
haproxy_executable = ${haproxy:location}/sbin/haproxy
rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
curl = ${curl:location}
dash = ${dash:location}
gzip = ${gzip:location}
logrotate = ${logrotate:location}
openssl = ${openssl:location}/bin/openssl
openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
trafficserver = ${trafficserver:location}
sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
xz_location = ${xz-utils:location}
monitor_template = ${monitor-template:output}
template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
template_graceful_script = ${template-graceful-script:target}
template_validate_script = ${template-validate-script:target}
template_rotate_script = ${template-rotate-script:target}
template_configuration_state_script = ${template-configuration-state-script:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
template_log_access = ${template-log-access:target}
template_not_found_html = ${template-not-found-html:target}
template_slave_list = ${template-slave-list:target}
template_trafficserver_records_config = ${template-trafficserver-records-config:target}
template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
template_wrapper = ${template-wrapper:output}
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
key common_profile template-common:rendered
key monitor2_template monitor2-template:rendered
key template_caddy_frontend template-caddy-frontend:target
key template_caddy_replicate template-caddy-replicate:target
key template_kedifa template-kedifa:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
key caucase_jinja2_library caucase-jinja2-library:target
[template-caddy-frontend]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
mode = 0644
[caddy-backend-url-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
mode = 0644
[template-kedifa]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-kedifa.cfg.in
mode = 0644
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[template-slave-list]
<=download-template
[template-replicate-publish-slave-information]
<=download-template
[template-caddy-frontend-configuration]
<=download-template
[template-not-found-html]
<=download-template
[template-default-slave-virtualhost]
<=download-template
[template-backend-haproxy-configuration]
<=download-template
[template-log-access]
<=download-template
[template-empty]
<=download-template
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
<=download-template
[template-trafficserver-storage-config]
<=download-template
[template-trafficserver-logging-config]
<=download-template
[template-rotate-script]
<=download-template
[template-caddy-lazy-script-call]
<=download-template
[template-graceful-script]
<=download-template
[template-validate-script]
<=download-template
[template-configuration-state-script]
<=download-template
[template-backend-haproxy-rsyslogd-conf]
<=download-template
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/development.cfg 0000664 0000000 0000000 00000001022 13712426147 0031112 0 ustar 00root root 0000000 0000000 # Development profile of caddy-frontend.
# Exactly the same as software.cfg, but fetch the slapos.cookbook
# from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
# You'll need to run buildout twice for this profile.
[buildout]
extends =
# Extend in this order, otherwise "parts" will be taken from git profile
common.cfg
parts +=
slapos.toolbox-dev
[slapos.toolbox-dev]
recipe = zc.recipe.egg:develop
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
instance-apache-frontend.cfg.in 0000664 0000000 0000000 00000103752 13712426147 0033773 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {%- if slap_software_type == software_type -%}
{% import "caucase" as caucase with context %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
[buildout]
extends =
{{ parameter_dict['common_profile'] }}
{{ parameter_dict['monitor_template'] }}
{{ parameter_dict['logrotate_base_instance'] }}
parts =
directory
logrotate-entry-caddy
caddy-frontend
switch-caddy-softwaretype
caucase-updater
caucase-updater-promise
backend-client-caucase-updater
backend-client-caucase-updater-promise
frontend-caddy-graceful
port-redirection
promise-frontend-caddy-configuration
promise-caddy-frontend-v4-https
promise-caddy-frontend-v4-http
promise-caddy-frontend-v6-https
promise-caddy-frontend-v6-http
trafficserver-launcher
trafficserver-reload
trafficserver-configuration-directory
trafficserver-records-config
trafficserver-remap-config
trafficserver-plugin-config
trafficserver-storage-config
trafficserver-ip-allow-config
trafficserver-logging-config
trafficserver-promise-listen-port
trafficserver-promise-cache-availability
cron-entry-logrotate-trafficserver
## Monitor for Caddy
monitor-base
monitor-ats-cache-stats-wrapper
monitor-traffic-summary-last-stats-wrapper
monitor-caddy-server-status-wrapper
monitor-verify-re6st-connectivity
backend-haproxy-rsyslogd-configuration
backend-haproxy-rsyslogd
logrotate-entry-backend-haproxy
backend-haproxy
backend-haproxy-graceful
promise-backend-haproxy-http
promise-backend-haproxy-https
promise-backend-haproxy-configuration
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin/
etc = ${buildout:directory}/etc/
srv = ${buildout:directory}/srv/
var = ${buildout:directory}/var/
tmp = ${:var}/tmp
template = ${buildout:directory}/template/
backup = ${:srv}/backup
log = ${:var}/log
run = ${:var}/run
backend-haproxy-rsyslogd-spool = ${:run}/backend-haproxy-rsyslogd-spool
service = ${:etc}/service
etc-run = ${:etc}/run
ca-dir = ${:srv}/ssl
backend-client-dir = ${:srv}/backend-client
# BBB: SlapOS Master non-zero knowledge BEGIN
bbb-ssl-dir = ${:srv}/bbb-ssl
# BBB: SlapOS Master non-zero knowledge END
frontend_cluster = ${:var}/frontend_cluster
# csr_id publication
csr_id = ${:srv}/csr_id
caddy-csr_id = ${:etc}/caddy-csr_id
caddy-csr_id-log = ${:log}/httpd-csr_id
[switch-caddy-softwaretype]
recipe = slapos.cookbook:softwaretype
single-default = ${dynamic-custom-personal-template-slave-list:rendered}
single-custom-personal = ${dynamic-custom-personal-template-slave-list:rendered}
[frontend-configuration]
template-log-access = {{ parameter_dict['template_log_access'] }}
log-access-configuration = ${directory:etc}/log-access.conf
ip-access-certificate = ${self-signed-ip-access:certificate}
caddy-directory = {{ parameter_dict['caddy_location'] }}
caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
caddy-https-port = ${configuration:port}
[self-signed-ip-access]
# Self Signed certificate for HTTPS IP accesses to the frontend
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
stop-on-error = True
command =
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:certificate} \
-subj "/CN=Self Signed IP Access" \
-reqexts SAN \
-extensions SAN \
-config <(cat {{ parameter_dict['openssl_cnf'] }} \
<(printf "\n[SAN]\nsubjectAltName=IP:${:ipv6},IP:${:ipv4}")) \
-out ${:certificate}'
[self-signed-fallback-access]
# Self Signed certificate for HTTPS access to the frontend with fallback certificate
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
stop-on-error = True
command =
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:certificate} \
-subj "/CN=Fallback certificate/OU={{ instance_parameter['configuration.frontend-name'] }}" \
-out ${:certificate}'
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
context =
import json_module json
raw common_profile {{ parameter_dict['common_profile'] }}
raw logrotate_base_instance {{ parameter_dict['logrotate_base_instance'] }}
raw monitor_template {{ parameter_dict['monitor_template'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
[software-release-path]
template-empty = {{ parameter_dict['template_empty'] }}
template-default-slave-virtualhost = {{ parameter_dict['template_default_slave_virtualhost'] }}
template-backend-haproxy-configuration = {{ parameter_dict['template_backend_haproxy_configuration'] }}
template-backend-haproxy-rsyslogd-conf = {{ parameter_dict['template_backend_haproxy_rsyslogd_conf'] }}
caddy-location = {{ parameter_dict['caddy_location'] }}
[kedifa-login-config]
d = ${directory:ca-dir}
template-csr = ${:d}/kedifa-login-template-csr.pem
key = ${:d}/kedifa-login-certificate.pem
certificate = ${:key}
ca-certificate = ${:d}/kedifa-caucase-ca.pem
cas-ca-certificate = ${:d}/kedifa-cas-caucase-ca.pem
crl = ${:d}/kedifa-login-crl.pem
[kedifa-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
command =
{% if slapparameter_dict['kedifa-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:template-csr}
fi
{% endif %}
test -f ${:key} && test -f ${:template-csr}
update-command = ${:command}
template-csr = ${kedifa-login-config:template-csr}
key = ${kedifa-login-config:key}
stop-on-error = True
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
updater_path='${directory:service}/kedifa-login-certificate-caucase-updater',
url=slapparameter_dict['kedifa-caucase-url'],
data_dir='${directory:srv}/caucase-updater',
crt_path='${kedifa-login-config:certificate}',
ca_path='${kedifa-login-config:ca-certificate}',
crl_path='${kedifa-login-config:crl}',
key_path='${kedifa-login-csr:key}',
template_csr='${kedifa-login-csr:template-csr}'
)}}
[kedifa-configuration]
caucase-url = {{ slapparameter_dict['kedifa-caucase-url'] }}
ca-certificate = ${kedifa-login-config:ca-certificate}
certificate = ${kedifa-login-config:certificate}
cas-ca-certificate = ${kedifa-login-config:cas-ca-certificate}
csr = ${caucase-updater-csr:csr}
crl = ${kedifa-login-config:crl}
kedifa-updater = {{ parameter_dict['kedifa-updater'] }}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }}
[backend-client-login-config]
d = ${directory:backend-client-dir}
template-csr = ${:d}/csr.pem
key = ${:d}/certificate.pem
certificate = ${:key}
ca-certificate = ${:d}/ca.pem
cas-ca-certificate = ${:d}/cas-ca.pem
crl = ${:d}/crl.pem
[backend-client-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
command =
{% if slapparameter_dict['backend-client-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:template-csr}
fi
{% endif %}
test -f ${:key} && test -f ${:template-csr}
update-command = ${:command}
template-csr = ${backend-client-login-config:template-csr}
key = ${backend-client-login-config:key}
stop-on-error = True
{{ caucase.updater(
prefix='backend-client-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
updater_path='${directory:service}/backend-client-login-certificate-caucase-updater',
url=slapparameter_dict['backend-client-caucase-url'],
data_dir='${directory:srv}/backend-client-caucase-updater',
crt_path='${backend-client-login-config:certificate}',
ca_path='${backend-client-login-config:ca-certificate}',
crl_path='${backend-client-login-config:crl}',
key_path='${backend-client-login-csr:key}',
template_csr='${backend-client-login-csr:template-csr}'
)}}
[dynamic-custom-personal-template-slave-list]
< = jinja2-template-base
template = {{ parameter_dict['template_slave_list'] }}
filename = custom-personal-instance-slave-list.cfg
slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
extra_slave_instance_list = {{ dumps(instance_parameter.get('configuration.extra_slave_instance_list')) }}
master_key_download_url = {{ dumps(slapparameter_dict['master-key-download-url']) }}
local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
local_ipv6 = {{ dumps(instance_parameter['ipv6-random']) }}
software_type = single-custom-personal
bin_directory = {{ parameter_dict['bin_directory'] }}
caddy_executable = {{ parameter_dict['caddy'] }}
sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational-unit = {{ instance_parameter['configuration.frontend-name'] }}
backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
extra-context =
key caddy_configuration_directory caddy-directory:slave-configuration
key backend_client_caucase_url :backend-client-caucase-url
import urlparse_module urlparse
key caddy_executable :caddy_executable
key http_port configuration:plain_http_port
key https_port configuration:port
key public_ipv4 configuration:public-ipv4
key slave_instance_list :slave_instance_list
key extra_slave_instance_list :extra_slave_instance_list
key master_key_download_url :master_key_download_url
key autocert caddy-directory:autocert
key master_certificate caddy-configuration:master-certificate
key caddy_log_directory caddy-directory:slave-log
key expose_csr_id_organization :organization
key expose_csr_id_organizational_unit :organizational-unit
key local_ipv4 :local_ipv4
key local_ipv6 :local_ipv6
key global_ipv6 slap-network-information:global-ipv6
key empty_template software-release-path:template-empty
key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
key software_type :software_type
key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
key frontend_graceful_reload caddy-configuration:frontend-graceful-command
section frontend_configuration frontend-configuration
section caddy_configuration caddy-configuration
key monitor_base_url monitor-instance-parameter:monitor-base-url
key bin_directory :bin_directory
key enable_http2_by_default configuration:enable-http2-by-default
key global_disable_http2 configuration:global-disable-http2
key ciphers configuration:ciphers
key access_log caddy-configuration:access-log
key error_log caddy-configuration:error-log
key sixtunnel_executable :sixtunnel_executable
key not_found_file caddy-configuration:not-found-file
key custom_ssl_directory caddy-directory:custom-ssl-directory
section kedifa_configuration kedifa-configuration
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
# BBB: SlapOS Master non-zero knowledge END
## backend haproxy
key template_backend_haproxy_configuration software-release-path:template-backend-haproxy-configuration
section backend_haproxy_configuration backend-haproxy-configuration
## full configuration
section configuration configuration
# Deploy Caddy Frontend with Jinja power
[dynamic-caddy-frontend-template]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_frontend_configuration'] }}
rendered = ${caddy-configuration:frontend-configuration}
local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
extra-context =
key httpd_home software-release-path:caddy-location
key httpd_mod_ssl_cache_directory caddy-directory:mod-ssl
key instance_home buildout:directory
key master_certificate caddy-configuration:master-certificate
key access_log caddy-configuration:access-log
key slave_configuration_directory caddy-directory:slave-configuration
section frontend_configuration frontend-configuration
key http_port configuration:plain_http_port
key https_port configuration:port
key local_ipv4 :local_ipv4
key global_ipv6 slap-network-information:global-ipv6
key error_log caddy-configuration:error-log
key not_found_file caddy-configuration:not-found-file
key username monitor-instance-parameter:username
key password monitor-htpasswd:passwd
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
# BBB: SlapOS Master non-zero knowledge END
[caddy-wrapper]
recipe = slapos.recipe.template:jinja2
template = inline:
#!/bin/sh
export CADDYPATH=${directory:frontend_cluster}
ulimit -n $(ulimit -Hn)
exec {{ parameter_dict['caddy'] }} \
-conf ${dynamic-caddy-frontend-template:rendered} \
-log ${caddy-configuration:error-log} \
-log-roll-mb 0 \
{% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
-http2=false \
{% else %}
-http2=true \
{% endif %}
-grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s \
-disable-http-challenge \
-disable-tls-alpn-challenge \
"$@"
rendered = ${directory:bin}/caddy-wrapper
mode = 0755
[caddy-frontend]
recipe = slapos.cookbook:wrapper
command-line = ${caddy-wrapper:rendered} -pidfile ${caddy-configuration:pid-file}
wrapper-path = ${directory:service}/frontend_caddy
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files = ${caddy-wrapper:rendered}
[not-found-html]
recipe = plone.recipe.command
update-command = ${:command}
filename = notfound.html
command = ln -sf {{ parameter_dict['template_not_found_html'] }} ${caddy-directory:document-root}/${:filename}
[caddy-directory]
recipe = slapos.cookbook:mkdirectory
document-root = ${directory:srv}/htdocs
slave-configuration = ${directory:etc}/caddy-slave-conf.d/
cache = ${directory:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
slave-log = ${directory:log}/httpd
autocert = ${directory:srv}/autocert
master-autocert-dir = ${:autocert}/master-autocert
custom-ssl-directory = ${:slave-configuration}/ssl
[caddy-configuration]
frontend-configuration = ${directory:etc}/Caddyfile
access-log = ${directory:log}/frontend-access.log
error-log = ${directory:log}/frontend-error.log
pid-file = ${directory:run}/httpd.pid
frontend-graceful-command = ${frontend-caddy-validate:rendered} && kill -USR1 $(cat ${:pid-file})
not-found-file = ${caddy-directory:document-root}/${not-found-html:filename}
master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
# Communication with ATS
cache-port = ${trafficserver-variable:input-port}
# BBB: SlapOS Master non-zero knowledge BEGIN
[get-self-signed-fallback-access]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${self-signed-fallback-access:certificate}
[apache-certificate]
recipe = slapos.recipe.template:jinja2
template = inline:
{% raw %}
{{ certificate or fallback_certificate }}
{{ key or '' }}
{% endraw %}
context =
key certificate configuration:apache-certificate
key key configuration:apache-key
key fallback_certificate get-self-signed-fallback-access:certificate
rendered = ${directory:bbb-ssl-dir}/frontend.crt
# BBB: SlapOS Master non-zero knowledge END
[logrotate-entry-caddy]
<= logrotate-entry-base
name = caddy
log = ${caddy-configuration:error-log} ${caddy-configuration:access-log}
rotate-num = ${configuration:rotate-num}
# Note: Slaves do not define their own reload, as this would be repeated,
# because sharedscripts work per entry, and each slave needs its own
# olddir
# Here we trust that there will be something to be rotated with error
# or access log, and that this will trigger postrotate script.
post = ${frontend-caddy-lazy-graceful:rendered} &
#################
# Trafficserver
#################
[trafficserver-directory]
recipe = slapos.cookbook:mkdirectory
configuration = ${directory:etc}/trafficserver
local-state = ${directory:var}/trafficserver
bin_path = {{ parameter_dict['trafficserver'] }}/bin
log = ${directory:log}/trafficserver
cache-path = ${directory:srv}/ats_cache
logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
[trafficserver-variable]
wrapper-path = ${directory:service}/trafficserver
reload-path = ${directory:etc-run}/trafficserver-reload
local-ip = {{ instance_parameter['ipv4-random'] }}
input-port = 23432
hostname = ${configuration:frontend-name}
plugin-config =
ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
cache-path = ${trafficserver-directory:cache-path}
disk-cache-size = ${configuration:disk-cache-size}
synthetic-port = ${configuration:trafficserver-synthetic-port}
mgmt-port = ${configuration:trafficserver-mgmt-port}
ram-cache-size = ${configuration:ram-cache-size}
templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
request-timeout = ${configuration:request-timeout}
[trafficserver-configuration-directory]
recipe = plone.recipe.command
command = cp -rn {{ parameter_dict['trafficserver'] }}/etc/trafficserver/* ${:target}
target = ${trafficserver-directory:configuration}
[trafficserver-launcher]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
wrapper-path = ${trafficserver-variable:wrapper-path}
environment = TS_ROOT=${buildout:directory}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[trafficserver-reload]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
wrapper-path = ${trafficserver-variable:reload-path}
environment = TS_ROOT=${buildout:directory}
# XXX Dedicated Jinja Section without slapparameter
[trafficserver-jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${trafficserver-directory:configuration}/${:filename}
extra-context =
mode = 600
context =
section ats_directory trafficserver-directory
section ats_configuration trafficserver-variable
${:extra-context}
[trafficserver-records-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_records_config'] }}
filename = records.config
extra-context =
import os_module os
[trafficserver-storage-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_storage_config'] }}
filename = storage.config
[trafficserver-logging-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_logging_config'] }}
filename = logging.config
[trafficserver-remap-config]
<= trafficserver-jinja2-template-base
{%- raw %}
template = inline:
map /HTTPS/ http://{{ ipv4 }}:{{ https_port }}
map / http://{{ ipv4 }}:{{ http_port }}
{%- endraw %}
extra-context =
raw ipv4 {{ instance_parameter['ipv4-random'] }}
key https_port backend-haproxy-configuration:https-port
key http_port backend-haproxy-configuration:http-port
filename = remap.config
[trafficserver-plugin-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
filename = plugin.config
context =
key content trafficserver-variable:plugin-config
[trafficserver-ip-allow-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
filename = ip_allow.config
context =
key content trafficserver-variable:ip-allow-config
[trafficserver-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
name = trafficserver-port-listening.py
config-hostname = ${trafficserver-variable:local-ip}
config-port = ${trafficserver-variable:input-port}
[trafficserver-ctl]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl
wrapper-path = ${directory:bin}/traffic_ctl
environment = TS_ROOT=${buildout:directory}
[trafficserver-promise-cache-availability]
<= monitor-promise-base
module = trafficserver_cache_availability
name = trafficserver-cache-availability.py
config-wrapper-path = ${trafficserver-ctl:wrapper-path}
[trafficserver-rotate-script]
< = jinja2-template-base
template = {{ parameter_dict['template_rotate_script'] }}
rendered = ${directory:bin}/trafficserver-rotate
mode = 0700
xz_binary = {{ parameter_dict['xz_location'] ~ '/bin/xz' }}
pattern = *.old
# days to keep log files
keep_days = 365
extra-context =
key log_dir trafficserver-directory:log
key rotate_dir trafficserver-directory:logrotate-backup
key xz_binary :xz_binary
key keep_days :keep_days
key pattern :pattern
[cron-entry-logrotate-trafficserver]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = trafficserver-logrotate
frequency = 0 0 * * *
command = ${trafficserver-rotate-script:rendered}
### End of ATS sections
### Caddy Graceful and promises
[frontend-caddy-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
extra-context =
key path_list :path_list
key sha256sum :sha256sum
key signature_file :signature_file
[frontend-caddy-configuration-state-graceful]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/graceful_configuration_state_signature
[frontend-caddy-configuration-state-validate]
< = frontend-caddy-configuration-state
signature_file = ${directory:run}/validate_configuration_state_signature
[frontend-caddy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
mode = 0700
extra-context =
key graceful_reload_command caddy-configuration:frontend-graceful-command
key caddy_configuration_state frontend-caddy-configuration-state-graceful:rendered
[frontend-caddy-validate]
< = jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/frontend-caddy-validate
mode = 0700
last_state_file = ${directory:run}/caddy_configuration_last_state
validate_command = ${caddy-wrapper:rendered} -validate
extra-context =
key validate_command :validate_command
key configuration_state_command frontend-caddy-configuration-state-validate:rendered
key last_state_file :last_state_file
[frontend-caddy-lazy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
rendered = ${directory:bin}/frontend-caddy-lazy-graceful
mode = 0700
pid-file = ${directory:run}/lazy-graceful.pid
wait_time = 60
extra-context =
key pid_file :pid-file
key wait_time :wait_time
key lazy_command caddy-configuration:frontend-graceful-command
# Promises checking configuration:
[promise-helper-last-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
rendered = ${directory:bin}/frontend-read-last-configuration-state
mode = 0700
content =
#!/bin/sh
exit `cat ${frontend-caddy-validate:last_state_file}`
context =
key content :content
[promise-frontend-caddy-configuration]
<= monitor-promise-base
module = validate_frontend_configuration
name = frontend-caddy-configuration-promise.py
config-verification-script = ${promise-helper-last-configuration-state:rendered}
[promise-caddy-frontend-v4-https]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv4_https.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v4-http]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv4_http.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-port = ${configuration:plain_http_port}
[promise-caddy-frontend-v6-https]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv6_https.py
config-hostname = {{ instance_parameter['ipv6-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v6-http]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv6_http.py
config-hostname = {{ instance_parameter['ipv6-random'] }}
config-port = ${configuration:plain_http_port}
[promise-backend-haproxy-http]
<= monitor-promise-base
module = check_port_listening
name = backend_haproxy_http.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:http-port}
[promise-backend-haproxy-https]
<= monitor-promise-base
module = check_port_listening
name = backend_haproxy_https.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:https-port}
[backend-haproxy-configuration]
file = ${directory:etc}/backend-haproxy.cfg
pid-file = ${directory:run}/backend-haproxy.pid
log-socket = ${backend-haproxy-rsyslogd-config:log-socket}
graceful-command = ${backend-haproxy-validate:rendered} && kill -USR2 $(cat ${:pid-file})
http-port = ${configuration:backend-haproxy-http-port}
https-port = ${configuration:backend-haproxy-https-port}
# Caucase related configuration
caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
ca-certificate = ${backend-client-login-config:ca-certificate}
certificate = ${backend-client-login-config:certificate}
cas-ca-certificate = ${backend-client-login-config:cas-ca-certificate}
csr = ${backend-client-caucase-updater-csr:csr}
crl = ${backend-client-login-config:crl}
[backend-haproxy]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file}
wrapper-path = ${directory:service}/backend-haproxy
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[backend-haproxy-rsyslogd-lazy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
rendered = ${directory:bin}/backend-haproxy-rsyslogd-lazy-graceful
mode = 0700
pid-file = ${directory:run}/backend-haproxy-rsyslogd-lazy-graceful.pid
wait_time = 60
extra-context =
key pid_file :pid-file
key wait_time :wait_time
key lazy_command backend-haproxy-rsyslogd-config:graceful-command
[logrotate-entry-backend-haproxy]
<= logrotate-entry-base
name = backend-haproxy
log = ${backend-haproxy-rsyslogd-config:log-file}
rotate-num = ${configuration:rotate-num}
# Note: Slaves do not define their own reload, as this would be repeated,
# because sharedscripts work per entry, and each slave needs its own
# olddir
# Here we trust that there will be something to be rotated with error
# or access log, and that this will trigger postrotate script.
post = ${backend-haproxy-rsyslogd-lazy-graceful:rendered} &
[backend-haproxy-configuration-state]
<= jinja2-template-base
template = {{ parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${backend-haproxy-configuration:file} ${backend-client-login-config:certificate}
sha256sum = {{ parameter_dict['sha256sum'] }}
extra-context =
key path_list :path_list
key sha256sum :sha256sum
key signature_file :signature_file
[backend-haproxy-configuration-state-graceful]
<= backend-haproxy-configuration-state
signature_file = ${directory:run}/backend_haproxy_graceful_configuration_state_signature
[backend-haproxy-configuration-state-validate]
<= backend-haproxy-configuration-state
signature_file = ${directory:run}/backend_haproxy_validate_configuration_state_signature
[backend-haproxy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/backend-haproxy-safe-graceful
mode = 0700
extra-context =
key graceful_reload_command backend-haproxy-configuration:graceful-command
key caddy_configuration_state backend-haproxy-configuration-state-graceful:rendered
[backend-haproxy-validate]
<= jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/backend-haproxy-validate
mode = 0700
last_state_file = ${directory:run}/backend_haproxy_configuration_last_state
validate_command = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file} -c
extra-context =
key validate_command :validate_command
key configuration_state_command backend-haproxy-configuration-state-validate:rendered
key last_state_file :last_state_file
[promise-backend-haproxy-configuration]
<= monitor-promise-base
module = validate_frontend_configuration
name = backend-haproxy-configuration.py
config-verification-script = ${promise-backend-haproxy-configuration-helper:rendered}
[promise-backend-haproxy-configuration-helper]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
rendered = ${directory:bin}/backend-haproxy-read-last-configuration-state
mode = 0700
content =
#!/bin/sh
exit `cat ${backend-haproxy-validate:last_state_file}`
context =
key content :content
[backend-haproxy-rsyslogd-config]
log-socket = ${directory:run}/bhlog.sck
log-file = ${directory:log}/backend-haproxy.log
pid-file = ${directory:run}/backend-haproxy-rsyslogd.pid
spool-directory = ${directory:backend-haproxy-rsyslogd-spool}
graceful-command = kill -HUP $(cat ${:pid-file})
caddy-log-directory = ${caddy-directory:slave-log}
[backend-haproxy-rsyslogd-configuration]
<= jinja2-template-base
template = ${software-release-path:template-backend-haproxy-rsyslogd-conf}
rendered = ${directory:etc}/backend-haproxy-rsyslogd.conf
extra-context =
section configuration backend-haproxy-rsyslogd-config
[backend-haproxy-rsyslogd]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['rsyslogd_executable'] }} -i ${backend-haproxy-rsyslogd-config:pid-file} -n -f ${backend-haproxy-rsyslogd-configuration:rendered}
wrapper-path = ${directory:service}/backend-haproxy-rsyslogd
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
#######
# Monitoring sections
#
[monitor-instance-parameter]
# Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
# directly, and in our case it can come from the network, thus resulting
# with need to strip !py!'u'
monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter['configuration.monitor-password'] | string }}
[monitor-conf-parameters]
private-path-list +=
${logrotate-directory:logrotate-backup}
[monitor-traffic-summary-last-stats-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/traffic-summary-last-stats_every_1_hour
mode = 0700
command = export TS_ROOT=${buildout:directory} && echo "$({{ parameter_dict['trafficserver'] }}/bin/traffic_logstats -f ${trafficserver-directory:log}/squid.blog)
"
extra-context =
key content monitor-traffic-summary-last-stats-wrapper:command
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/ats-cache-stats_every_1_hour
mode = 0700
command = export TS_ROOT=${buildout:directory} && echo "$({{ parameter_dict['trafficserver'] }}/bin/traffic_shell ${monitor-ats-cache-stats-config:rendered})
"
extra-context =
key content monitor-ats-cache-stats-wrapper:command
[monitor-caddy-server-status-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/monitor-caddy-server-status-wrapper
mode = 0700
command = {{ parameter_dict['curl'] }}/bin/curl -s http://{{ instance_parameter['ipv4-random'] }}:${configuration:plain_http_port}/server-status -u ${monitor-instance-parameter:username}:${monitor-htpasswd:passwd} 2>&1
extra-context =
key content monitor-caddy-server-status-wrapper:command
[monitor-ats-cache-stats-config]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
rendered = ${trafficserver-configuration-directory:target}/cache-config.stats
mode = 644
context =
raw content show:cache-stats
[monitor-verify-re6st-connectivity]
<= monitor-promise-base
module = check_url_available
name = re6st-connectivity.py
config-url = ${configuration:re6st-verification-url}
[port-redirection]
<= jinja2-template-base
template = inline:
[{"srcPort": 80, "destPort": {{ '{{' }} http_port {{ '}}' }}}, {"srcPort": 443, "destPort": {{ '{{' }} https_port {{ '}}' }}}]
rendered = ${buildout:directory}/.slapos-port-redirect
mode = 0644
extra-context =
key http_port configuration:plain_http_port
key https_port configuration:port
[configuration]
{%- for key, value in instance_parameter.iteritems() -%}
{%- if key.startswith('configuration.') %}
{{ key.replace('configuration.', '') }} = {{ dumps(value) }}
{%- endif -%}
{%- endfor -%}
{%- endif -%} {# if slap_software_type == software_type #}
instance-apache-replicate.cfg.in 0000664 0000000 0000000 00000076674 13712426147 0034140 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {% if slap_software_type in software_type %}
{% set aibcc_enabled = True %}
{% import "caucase" as caucase with context %}
{#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
{%- set SERVER_POLLUTED_KEY_LIST = ['connection-parameter-hash', 'timestamp', 'slave_title', 'slap_software_type'] -%}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{%- set GOOD_CIPHER_LIST = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-AES256-CBC-SHA', 'ECDHE-RSA-AES128-CBC-SHA', 'ECDHE-ECDSA-AES256-CBC-SHA', 'ECDHE-ECDSA-AES128-CBC-SHA', 'RSA-AES256-CBC-SHA', 'RSA-AES128-CBC-SHA', 'ECDHE-RSA-3DES-EDE-CBC-SHA', 'RSA-3DES-EDE-CBC-SHA'] %}
{% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
{% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
{# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
{% set master_partition_monitor_monitor_httpd_port = 8401 %}
{% set kedifa_partition_monitor_httpd_port = 8402 %}
{% set frontend_monitor_httpd_base_port = 8410 %}
{% set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_backend_client_port'] %}
{% set caucase_url = 'http://' ~ caucase_netloc %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
raw common_profile {{ common_profile }}
${:extra-context}
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %}
{% set single_type_key = 'single-' %}
{% if slap_software_type == "replicate" %}
{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
{% elif slap_software_type in ['default', 'RootSoftwareInstance'] %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% else %}
{% set frontend_type = "%s%s" % (single_type_key, slap_software_type) %}
{% endif %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %}
{% set frontend_list = [] %}
{% set frontend_section_list = [] %}
{% set request_dict = {} %}
{% set namebase = 'caddy-frontend' %}
# XXX Dirty hack, not possible to define default value before
{% set sla_computer_caddy_1_key = '-sla-1-computer_guid' %}
{% if not sla_computer_caddy_1_key in slapparameter_dict %}
{% do slapparameter_dict.__setitem__(sla_computer_caddy_1_key, '${slap-connection:computer-id}') %}
{% endif %}
{% set sla_computer_kedifa_key = '-sla-kedifa-computer_guid' %}
{% if not sla_computer_kedifa_key in slapparameter_dict %}
{% do slapparameter_dict.__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %}
{% endif %}
# Here we request individually each frontend.
# The presence of sla parameters is checked and added if found
{% for i in range(1, frontend_quantity + 1) %}
{% set frontend_name = "%s-%s" % (namebase, i) %}
{% set request_section_title = 'request-%s' % frontend_name %}
{% set sla_key = "-sla-%s-" % i %}
{% set sla_key_length = sla_key | length %}
{% set sla_dict = {} %}
{% set config_key = "-frontend-config-%s-" % i %}
{% set config_key_length = config_key | length %}
{% set config_dict = {} %}
{% for key in slapparameter_dict.keys() %}
{% if key.startswith(sla_key) %}
{% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
# We check for specific configuration regarding the frontend
{% elif key.startswith(config_key) %}
{% do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %}
{% endif %}
{% endfor %}
{% do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %}
{% do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %}
{% do frontend_list.append(frontend_name) %}
{% do frontend_section_list.append(request_section_title) %}
{% do part_list.append(request_section_title) %}
# Filling request dict for slave
{% set state_key = "-frontend-%s-state" % i %}
{% do request_dict.__setitem__(request_section_title,
{
'config': config_dict,
'name': frontend_name,
'sla': sla_dict,
'state': slapparameter_dict.pop(state_key, None)
}) %}
{% endfor %}
{% set authorized_slave_string_list = [] %}
{% set authorized_slave_list = [] %}
{% set rejected_slave_dict = {} %}
{% set rejected_slave_title_dict = {} %}
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% for slave in sorted(slave_instance_list) %}
{% set slave_error_list = [] %}
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
{% set slave_type = slave.get('type') %}
{% if slave_type == 'eventsource' %}
{% do slave_error_list.append('type:eventsource is not implemented') %}
{% elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{% endif %}
{# Check ciphers #}
{% set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
{% if slave_cipher_list %}
{% for cipher in slave_cipher_list %}
{% if cipher not in GOOD_CIPHER_LIST %}
{% do slave_error_list.append('Cipher %r is not supported.' % (cipher,)) %}
{% endif %}
{% endfor %}
{% endif %}
{% set custom_domain = slave.get('custom_domain') %}
{% if custom_domain and custom_domain in used_host_list %}
{% do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %}
{% else %}
{% do used_host_list.append(custom_domain) %}
{% endif %}
{% if slave.get('server-alias') %}
{% for slave_alias in ('' ~ slave['server-alias']).split() %}
{% if slave_alias.startswith('*.') %}
{% set clean_slave_alias = slave_alias[2:] %}
{% else %}
{% set clean_slave_alias = slave_alias %}
{% endif %}
{% if not validators.domain(clean_slave_alias) %}
{% do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %}
{% else %}
{% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
{# optionally do something about reporting back that server-alias has been unclashed #}
{% elif slave_alias in used_host_list %}
{% do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %}
{% else %}
{% do slave_server_alias_unclashed.append(slave_alias) %}
{% do used_host_list.append(slave_alias) %}
{% endif %}
{% endif %}
{% endfor %}
{% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
{% endif %}
{% for url_key in ['url', 'https-url'] %}
{% if url_key in slave %}
{% set url = (slave[url_key] or '').strip() %}
{% if subprocess_module.call([caddy_backend_url_validator, url]) == 1 or not validators.url(url) %}
{% do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
{% elif url != slave[url_key] %}
{% do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
{% endif %}
{% endif %}
{% endfor %}
{% if 'ssl_proxy_ca_crt' in slave %}
{% set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
{% set check_popen = popen([parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(ssl_proxy_ca_crt) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
{% endif %}
{% endif %}
{# BBB: SlapOS Master non-zero knowledge BEGIN #}
{% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{% if key in slave %}
{% do slave_warning_list.append('%s is obsolete, please use key-upload-url' % (key,)) %}
{% endif %}
{% endfor %}
{% if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %}
{% do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required') %}
{% endif %}
{% if slave.get('ssl_key') and slave.get('ssl_crt') %}
{% set key_popen = popen([parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{% set crt_popen = popen([parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
{% set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
{% set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
{% if not key_modulus or key_modulus != crt_modulus %}
{% do slave_error_list.append('slave ssl_key and ssl_crt does not match') %}
{% endif %}
{% endif %}
{# BBB: SlapOS Master non-zero knowledge END #}
{% if slave.get('custom_domain') %}
{% set slave_custom_domain = '' ~ slave['custom_domain'] %}
{% if slave_custom_domain.startswith('*.') %}
{% set clean_custom_domain = slave_custom_domain[2:] %}
{% else %}
{% set clean_custom_domain = slave_custom_domain %}
{% endif %}
{% if not validators.domain(clean_custom_domain) %}
{% do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %}
{% endif %}
{% endif %}
{% if len(slave_error_list) == 0 %}
{# Cleanup slave from not needed keys which come from implementation of SlapOS Master #}
{% set authorized_slave = slave.copy() %}
{% for key in SERVER_POLLUTED_KEY_LIST %}
{% do authorized_slave.pop(key, None) %}
{% endfor %}
{% do authorized_slave_list.append(authorized_slave) %}
{% else %}
{% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), slave_error_list) %}
{% do rejected_slave_title_dict.__setitem__(slave.get('slave_title'), slave_error_list) %}
{% endif %}
{% if len(slave_warning_list) > 0 %}
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), slave_warning_list) %}
{% endif %}
{% endfor %}
{% do authorized_slave_list.sort() %}
[monitor-instance-parameter]
monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }}
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
{% else %}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate
{% for section, frontend_request in request_dict.iteritems() %}
{% set state = frontend_request.get('state', '') %}
[{{section}}]
<= replicate
name = {{ frontend_request.get('name') }}
{% if state %}
state = {{ state }}
{% endif %}
config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information}
config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
config-backend-client-caucase-url = {{ caucase_url }}
config-master-key-download-url = ${request-kedifa:connection-master-key-download-url}
config-cluster-identification = {{ cluster_identification }}
{# Do not send additional parameters for destroyed nodes #}
{% if state != 'destroyed' %}
{% set slave_configuration_dict = slapparameter_dict %}
{% do slave_configuration_dict.update(frontend_request.get('config')) %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
{% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
{% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{% endif %}
{% if frontend_request.get('sla') %}
{% for parameter, value in frontend_request.get('sla').iteritems() %}
sla-{{ parameter }} = {{ value }}
{% endfor %}
{% endif %}
{% endfor %}
{% set warning_list = [] %}
{% for key in ['apache-certificate', 'apache-key'] %}
{% if key in slapparameter_dict %}
{% do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
{% endif %}
{% endfor %}
[publish-information]
<= monitor-publish
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_dict | length }}
backend-client-caucase-url = {{ caucase_url }}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_title_dict, sort_keys=True)) }}
rejected-slave-promise-url = ${rejected-slave-promise:config-url}
master-key-upload-url = ${request-kedifa:connection-master-key-upload-url}
master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url}
kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
{% if len(warning_list) > 0 %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
warning-list = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }}
{% endif %}
{% if len(warning_slave_dict) > 0 %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
warning-slave-dict = {{ dumps(json_module.dumps(warning_slave_dict, sort_keys=True)) }}
{% endif %}
{% if not aikc_enabled or not aibcc_enabled %}
{% for frontend in frontend_list %}
{% set section_part = '${request-' + frontend %}
{{ frontend }}-csr_id-certificate = {{ section_part }}:connection-csr_id-certificate}
{% endfor %}
{% endif %}
{% if not aikc_enabled %}
kedifa-csr_id-url = ${request-kedifa:connection-csr_id-url}
kedifa-csr_id-certificate = ${request-kedifa:connection-csr_id-certificate}
{% for frontend in frontend_list %}
{% set section_part = '${request-' + frontend %}
{{ frontend }}-csr_id-url = {{ section_part }}:connection-csr_id-url}
{% endfor %}
{% endif %}
{% if not aibcc_enabled %}
{% for frontend in frontend_list %}
{% set section_part = '${request-' + frontend %}
{{ frontend }}-backend-client-csr_id-url = {{ section_part }}:connection-backend-client-csr_id-url}
{% endfor %}
{% endif %}
#----------------------------
#--
#-- Publish slave information
[publish-slave-information]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-publish-slave-information:rendered}
RootSoftwareInstance = ${dynamic-publish-slave-information:rendered}
replicate = ${dynamic-publish-slave-information:rendered}
custom-personal = ${dynamic-publish-slave-information:rendered}
custom-group = ${dynamic-publish-slave-information:rendered}
[request-kedifa]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
config-monitor-httpd-port = {{ kedifa_partition_monitor_httpd_port }}
{% for key in ['kedifa_port', 'caucase_port'] -%}
{%- if key in slapparameter_dict %}
config-{{ key }} = {{ dumps(slapparameter_dict[key]) }}
{%- endif %}
{%- endfor %}
config-slave-list = {{ dumps(authorized_slave_list) }}
config-cluster-identification = {{ cluster_identification }}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
{% else %}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = kedifa
name = kedifa
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url csr_id-url csr_id-certificate monitor-base-url
{% set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in slapparameter_dict.keys() %}
{% if key.startswith(sla_kedifa_key) %}
sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
{% endif %}
{% endfor %}
[rejected-slave-information]
{% for slave_id, rejected_list in rejected_slave_dict.iteritems() %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
{{ slave_id }} = {{ dumps(json_module.dumps(rejected_list, sort_keys=True)) }}
{% endfor %}
[warning-slave-information]
{% for slave_id, warning_list in warning_slave_dict.iteritems() %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
{{ slave_id }} = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }}
{% endfor %}
[slave-information]
{% for frontend_section in frontend_section_list %}
{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
{% endfor %}
[active-slave-instance]
{% set active_slave_instance_list = [] %}
{% for slave_instance in slave_instance_list %}
{# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
{% do active_slave_instance_list.append(slave_instance['slave_reference']) %}
{% endfor %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }}
[dynamic-publish-slave-information]
< = jinja2-template-base
template = {{ template_publish_slave_information }}
filename = dynamic-publish-slave-information.cfg
extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
section rejected_slave_information rejected-slave-information
section active_slave_instance_dict active-slave-instance
section warning_slave_information warning-slave-information
key slave_kedifa_information request-kedifa:connection-slave-kedifa-information
[monitor-base-url-dict]
kedifa = ${request-kedifa:connection-monitor-base-url}
{% for frontend in frontend_section_list %}
{{ frontend }} = {{ '${' + frontend + ':connection-monitor-base-url}' }}
{% endfor %}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin/
srv = ${buildout:directory}/srv/
backup = ${:srv}/backup
# CAUCASE directories
caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased
{% if aikc_enabled %}
[directory]
aikc = ${:srv}/aikc
[aikc-config]
caucase-url = ${request-kedifa:connection-caucase-url}
csr = ${directory:aikc}/csr.pem
key = ${directory:aikc}/key.pem
ca-certificate = ${directory:aikc}/cas-ca-certificate.pem
crl = ${directory:aikc}/crl.pem
user-ca-certificate = ${directory:aikc}/user-ca-certificate.pem
user-crl = ${directory:aikc}/user-crl.pem
user-created = ${directory:aikc}/user-created
csr_id = ${directory:aikc}/csr_id
[aikc-user-csr]
recipe = plone.recipe.command
organization = {{ cluster_identification }}
organizational_unit = Automatic Internal Kedifa Caucase CSR
command =
if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:csr}
fi
update-command = ${:command}
csr = ${aikc-config:csr}
key = ${aikc-config:key}
stop-on-error = True
[aikc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
key caucase_url aikc-config:caucase-url
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
exec {{ parameter_dict['bin_directory'] }}/caucase \
{# raw block to use context #}
{% raw %}
--ca-url {{ caucase_url }} \
{% endraw %}
--ca-crt ${aikc-config:ca-certificate} \
--user-ca-crt ${aikc-config:user-ca-certificate} \
--user-crl ${aikc-config:user-crl} \
--crl ${aikc-config:crl} \
"$@"
rendered = ${directory:bin}/aikc-caucase-wrapper
mode = 0700
{% do part_list.append('aikc-create-user') %}
[aikc-create-user]
recipe = plone.recipe.command
stop-on-error = True
update-command = ${:command}
command =
if ! [ -f ${aikc-config:user-created} ] ; then
${aikc-caucase-wrapper:rendered} --mode user --send-csr ${aikc-user-csr:csr} > ${aikc-config:csr_id} || exit 1
cut -d ' ' -f 1 ${aikc-config:csr_id} || exit 1
csr_id=`cut -d ' ' -f 1 ${aikc-config:csr_id}`
sleep 1
${aikc-caucase-wrapper:rendered} --mode user --get-crt $csr_id ${aikc-config:key} || exit 1
touch ${aikc-config:user-created}
fi
{% do part_list.append('aikc-user-caucase-updater') %}
{% do part_list.append('aikc-user-caucase-updater-promise') %}
{{ caucase.updater(
prefix='aikc-user-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
updater_path='${directory:service}/aikc-user-caucase-updater',
url='${aikc-config:caucase-url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${aikc-config:key}',
ca_path='${aikc-config:user-ca-certificate}',
crl_path='${aikc-config:user-crl}',
key_path='${aikc-config:key}',
mode='user',
)}}
[aikc-check-certificate]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/aikc-check-certificate
template = inline:
import sys
import ssl
import urlparse
certificate = sys.argv[2]
parsed = urlparse.urlparse(sys.argv[1])
got_certificate = ssl.get_server_certificate((parsed.hostname, parsed.port))
sys.exit(0) if certificate.strip() == got_certificate.strip() else sys.exit(1)
{% for csr in frontend_list + ['kedifa'] %}
[aikc-{{ csr }}-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
key csr_id_url request-{{ csr }}:connection-csr_id-url
key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
test -f ${directory:aikc}/{{ csr }}-done && exit 0
${buildout:executable} ${aikc-check-certificate:rendered} \
{# raw block to use context #}
{% raw %}
{{ csr_id_url }} \
"""{{ csr_id_certificate }}"""
{% endraw %}
if [ $? = 0 ]; then
csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
{% raw %}
{{ csr_id_url }} \
{% endraw %}
` || exit 1
${aikc-caucase-wrapper:rendered} --user-key ${aikc-config:key} --sign-csr $csr_id && touch ${directory:aikc}/{{ csr }}-done
fi
rendered = ${directory:bin}/aikc-{{ csr }}-wrapper
mode = 0700
{% do part_list.append('aikc-%s' % (csr,)) %}
[aikc-{{ csr }}]
recipe = plone.recipe.command
stop-on-error = True
command =
${aikc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
{% endfor %}
{% endif %} {# if aikc_enabled #}
{% if aibcc_enabled %}
[directory]
aibcc = ${:srv}/aibcc
[aibcc-config]
caucase-url = {{ caucase_url }}
csr = ${directory:aibcc}/csr.pem
key = ${directory:aibcc}/key.pem
ca-certificate = ${directory:aibcc}/cas-ca-certificate.pem
crl = ${directory:aibcc}/crl.pem
user-ca-certificate = ${directory:aibcc}/user-ca-certificate.pem
user-crl = ${directory:aibcc}/user-crl.pem
user-created = ${directory:aibcc}/user-created
csr_id = ${directory:aibcc}/csr_id
[aibcc-user-csr]
recipe = plone.recipe.command
organization = {{ cluster_identification }}
organizational_unit = Automatic Sign Backend Client Caucase CSR
command =
if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:csr}
fi
update-command = ${:command}
csr = ${aibcc-config:csr}
key = ${aibcc-config:key}
stop-on-error = True
[aibcc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
key caucase_url aibcc-config:caucase-url
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
exec {{ parameter_dict['bin_directory'] }}/caucase \
{# raw block to use context #}
{% raw %}
--ca-url {{ caucase_url }} \
{% endraw %}
--ca-crt ${aibcc-config:ca-certificate} \
--user-ca-crt ${aibcc-config:user-ca-certificate} \
--user-crl ${aibcc-config:user-crl} \
--crl ${aibcc-config:crl} \
"$@"
rendered = ${directory:bin}/aibcc-caucase-wrapper
mode = 0700
{% do part_list.append('aibcc-create-user') %}
[aibcc-create-user]
recipe = plone.recipe.command
# the caucase for this part is provided in this profile, so we can't fail
# as otherwise caucase will never be started...
stop-on-error = False
update-command = ${:command}
command =
if ! [ -f ${aibcc-config:user-created} ] ; then
${aibcc-caucase-wrapper:rendered} --mode user --send-csr ${aibcc-user-csr:csr} > ${aibcc-config:csr_id} || exit 1
cut -d ' ' -f 1 ${aibcc-config:csr_id} || exit 1
csr_id=`cut -d ' ' -f 1 ${aibcc-config:csr_id}`
sleep 1
${aibcc-caucase-wrapper:rendered} --mode user --get-crt $csr_id ${aibcc-config:key} || exit 1
touch ${aibcc-config:user-created}
fi
{% do part_list.append('aibcc-user-caucase-updater') %}
{% do part_list.append('aibcc-user-caucase-updater-promise') %}
{{ caucase.updater(
prefix='aibcc-user-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
updater_path='${directory:service}/aibcc-user-caucase-updater',
url='${aibcc-config:caucase-url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${aibcc-config:key}',
ca_path='${aibcc-config:user-ca-certificate}',
crl_path='${aibcc-config:user-crl}',
key_path='${aibcc-config:key}',
mode='user',
)}}
[aibcc-check-certificate]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/aibcc-check-certificate
template = inline:
import sys
import ssl
import urlparse
certificate = sys.argv[2]
parsed = urlparse.urlparse(sys.argv[1])
got_certificate = ssl.get_server_certificate((parsed.hostname, parsed.port))
sys.exit(0) if certificate.strip() == got_certificate.strip() else sys.exit(1)
{% for csr in frontend_list %}
[aibcc-{{ csr }}-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2
context =
key csr_id_url request-{{ csr }}:connection-backend-client-csr_id-url
key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
test -f ${directory:aibcc}/{{ csr }}-done && exit 0
${buildout:executable} ${aibcc-check-certificate:rendered} \
{# raw block to use context #}
{% raw %}
{{ csr_id_url }} \
"""{{ csr_id_certificate }}"""
{% endraw %}
if [ $? = 0 ]; then
csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
{% raw %}
{{ csr_id_url }} \
{% endraw %}
` || exit 1
${aibcc-caucase-wrapper:rendered} --user-key ${aibcc-config:key} --sign-csr $csr_id && touch ${directory:aibcc}/{{ csr }}-done
fi
rendered = ${directory:bin}/aibcc-{{ csr }}-wrapper
mode = 0700
{% do part_list.append('aibcc-%s' % (csr,)) %}
[aibcc-{{ csr }}]
recipe = plone.recipe.command
stop-on-error = True
command =
${aibcc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
{% endfor %}
{% endif %} {# if aibcc_enabled #}
[rejected-slave-json]
recipe = slapos.recipe.template:jinja2
filename = rejected-slave.json
directory = ${directory:promise-output}
rendered = ${:directory}/${:filename}
template = {{ parameter_dict['template_empty'] }}
{% if rejected_slave_title_dict %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
content = {{ dumps(json_module.dumps(rejected_slave_title_dict, indent=2, sort_keys=True)) }}
{% else %}
content =
{% endif %}
context =
key content :content
[directory]
service = ${:etc}/service
promise-output = ${:srv}/promise-output
[rejected-slave-publish-configuration]
ip = {{ instance_parameter['ipv6-random'] }}
port = 14455
[rejected-slave-publish]
directory = ${rejected-slave-json:directory}
url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename}
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['caddy'] }}
-conf ${rejected-slave-template:rendered}
-log stderr
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root ${:directory}
wrapper-path = ${directory:service}/rejected-slave-publish
hash-existing-files =
${buildout:directory}/software_release/buildout.cfg
hash-files =
${rejected-slave-template:rendered}
${rejected-slave-certificate:certificate}
[rejected-slave-certificate]
recipe = plone.recipe.command
certificate = ${directory:etc}/rejected-slave.pem
key = ${:certificate}
stop-on-error = True
update-command = ${:command}
command =
[ -f ${:certificate} ] && find ${:certificate} -type f -mtime +3 -delete
if ! [ -f ${:certificate} ] ; then
openssl req -new -newkey rsa:2048 -sha256 -subj \
"/CN=${rejected-slave-publish-configuration:ip}" \
-days 5 -nodes -x509 -keyout ${:certificate} -out ${:certificate}
fi
[rejected-slave-password]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.rejected-slave.passwd
bytes = 8
user = admin
[rejected-slave-template]
recipe = slapos.recipe.template:jinja2
template = inline:
https://:${rejected-slave-publish-configuration:port}/ {
basicauth / ${rejected-slave-password:user} ${rejected-slave-password:passwd}
tls ${rejected-slave-certificate:certificate} ${rejected-slave-certificate:key}
bind ${rejected-slave-publish-configuration:ip}
log stderr
errors stderr
}
rendered = ${directory:etc}/Caddyfile-rejected-slave
[promise-rejected-slave-publish-ip-port]
<= monitor-promise-base
module = check_port_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}
[rejected-slave-promise]
<= monitor-promise-base
module = check_port_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
config-state = empty
config-url = ${rejected-slave-publish:url}
[caucased-backend-client]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{ caucase.caucased(
prefix='caucased-backend-client',
buildout_bin_directory=parameter_dict['bin_directory'],
caucased_path='${directory:service}/caucased-backend-client',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_netloc,
service_auto_approve_count=0,
user_auto_approve_count=1,
key_len=2048,
)}}
[buildout]
extends =
{{ common_profile }}
{{ template_monitor }}
parts =
monitor-base
publish-slave-information
publish-information
request-kedifa
rejected-slave-promise
promise-rejected-slave-publish-ip-port
caucased-backend-client
caucased-backend-client-promise
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
# publish-information
{% endif %}
instance-caddy-input-schema.json 0000664 0000000 0000000 00000011501 13712426147 0034207 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"-frontend-quantity": {
"description": "Quantity of Frontends Replicate.",
"title": "Frontend Replication Quantity",
"type": "integer"
},
"apache-certificate": {
"description": "SSL Certificate used by the server. By appending to it CA certificate it is possible to use this field to replace not implemented apache-ca-certificate. Deprecated, please use master-key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate, with optional CA certificate",
"type": "string"
},
"apache-key": {
"description": "SSL Key used by the server. Deprecated, please use master-key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Key",
"type": "string"
},
"domain": {
"description": "Base Domain for create subdomains (ie.: example.com).",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "Domain",
"type": "string"
},
"enable-http2-by-default": {
"default": "true",
"description": "Use HTTP2 as default Protocol",
"enum": [
"true",
"false"
],
"title": "Enable HTTP2 by Default",
"type": "string"
},
"global-disable-http2": {
"default": "false",
"description": "Disables globally HTTP2 in Caddy, thus enable-http2-by-default here and enable-http2 have no effect. Rational is that in some loaded environments it is better to run Caddy without any HTTP2 capabilities. Warning: Changing the parameter will result in restarting Caddy process.",
"enum": [
"true",
"false"
],
"title": "Globally disable HTTP2",
"type": "string"
},
"mpm-graceful-shutdown-timeout": {
"default": 5,
"description": "Value passed to -grace parameter of Caddy, see https://caddyserver.com/docs/cli .",
"title": "Duration of the graceful shutdown period. Warning: Changing the parameter will result in restarting Caddy process.",
"type": "integer"
},
"nginx-domain": {
"description": "Base Domain for create subdomains (ie.: example2.com) for websocket, notebook and eventsource.",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "[NOT IMPLEMENTED] Nginx Domain",
"type": "string"
},
"public-ipv4": {
"description": "Public ipv4 of the frontend (the one Caddy will be indirectly listening to).",
"title": "Public IPv4",
"type": "string"
},
"re6st-verification-url": {
"description": "Url to verify if the internet and/or re6stnet is working.",
"title": "Test Verification URL",
"type": "string"
},
"backend-connect-timeout": {
"default": 5,
"description": "Time in seconds for establishing connection to the backend.",
"title": "Timeout for backend connection (seconds)",
"type": "integer"
},
"backend-connect-retries": {
"default": 3,
"description": "Amount of retries to connect to the backend. The amount of backend-connect-timeout*backend-connect-retries seconds will be spent to connect to the backend.",
"title": "Amount of retries to connect to the backend.",
"type": "integer"
},
"automatic-internal-kedifa-caucase-csr": {
"default": "true",
"description": "Automatically signs CSRs sent to KeDiFa's caucase, based on csr_id and matching certificate.",
"enum": [
"true",
"false"
],
"title": "Automatic Internal KeDiFa's Caucase CSR",
"type": "string"
},
"automatic-internal-backend-client-caucase-csr": {
"default": "true",
"description": "Automatically signs CSRs sent to Backend Client's caucase, based on csr_id and matching certificate.",
"enum": [
"true",
"false"
],
"title": "Automatic Internal Backend Client's Caucase CSR",
"type": "string"
},
"ciphers": {
"description": "List of ciphers. Empty defaults to Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
"title": "Ordered space separated list of ciphers",
"type": "string"
},
"request-timeout": {
"default": 600,
"description": "Timeout for HTTP requests.",
"title": "HTTP Request timeout in seconds",
"type": "integer"
},
"authenticate-to-backend": {
"default": "false",
"description": "If set to true the frontend certificate will be used as authentication certificate to the backend. Note: backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
"enum": [
"false",
"true"
],
"title": "Authenticate to backend",
"type": "string"
}
},
"title": "Input Parameters",
"type": "object"
}
instance-common.cfg.in 0000664 0000000 0000000 00000000701 13712426147 0032213 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend [buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap_connection]
# Kept for backward compatibility
computer_id = ${slap-connection:computer-id}
partition_id = ${slap-connection:partition-id}
server_url = ${slap-connection:server-url}
software_release_url = ${slap-connection:software-release-url}
key_file = ${slap-connection:key-file}
cert_file = ${slap-connection:cert-file}
instance-kedifa.cfg.in 0000664 0000000 0000000 00000025704 13712426147 0032160 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {%- if slap_software_type == software_type -%}
{% import "caucase" as caucase with context %}
# KeDiFa instance profile
[buildout]
extends =
{{ parameter_dict['common_profile'] }}
{{ parameter_dict['monitor_template'] }}
{{ parameter_dict['logrotate_base_instance'] }}
parts =
monitor-base
directory
kedifa
logrotate-entry-kedifa
promise-kedifa-http-reply
slave-kedifa-information
caucased
caucased-promise
caucase-updater
expose-csr_id
promise-expose-csr_id-ip-port
[monitor-instance-parameter]
# Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
# directly, and in our case it can come from the network, thus resulting
# with need to strip !py!'u'
monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter['configuration.monitor-password'] | string }}
[caucased]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_port'] -%}
{% set caucase_url = 'http://' ~ caucase_netloc -%}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=parameter_dict['bin_directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
netloc=caucase_netloc,
service_auto_approve_count=0,
user_auto_approve_count=1,
key_len=2048,
)}}
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin/
etc = ${buildout:directory}/etc/
srv = ${buildout:directory}/srv/
var = ${buildout:directory}/var/
tmp = ${buildout:directory}/tmp/
backup = ${:srv}/backup
log = ${:var}/log
run = ${:var}/run
service = ${:etc}/service
etc-run = ${:etc}/run
# KeDiFa directories
kedifa = ${:srv}/kedifa
etc-kedifa = ${:etc}/kedifa
# CAUCASE directories
caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased
# reservation
reservation = ${:srv}/reservation
# csr_id publication
csr_id = ${:srv}/csr_id
caddy-csr_id = ${:etc}/caddy-csr_id
[kedifa-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = Kedifa Partition
command =
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
/bin/bash -c '{{ parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-reqexts SAN \
-config <(cat {{ parameter_dict['openssl_cnf'] }} \
<(printf "\n[SAN]\nsubjectAltName=IP:${kedifa-config:ip}")) \
-out ${:template-csr}'
fi
update-command = ${:command}
template-csr = ${kedifa-config:template-csr}
key = ${kedifa-config:key}
stop-on-error = True
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
updater_path='${directory:service}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
crt_path='${kedifa-config:certificate}',
ca_path='${kedifa-config:ca-certificate}',
crl_path='${kedifa-config:crl}',
key_path='${kedifa-csr:key}',
on_renew='${kedifa-reloader:wrapper-path}',
template_csr='${kedifa-csr:template-csr}'
)}}
[store-csr_id]
recipe = plone.recipe.command
csr_id_path = ${directory:csr_id}/csr_id.txt
csr_work_path = ${directory:tmp}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ parameter_dict['bin_directory'] }}/caucase \
--ca-url {{ caucase_url }} \
--ca-crt ${kedifa-config:ca-certificate} \
--crl ${kedifa-config:crl} \
--mode service \
{#- XXX: Need to use caucase-updater-csr:csr, as there is no way to obatin csr_id from caucase-updater -#}
{#- XXX: nor directly path to the generated CSR #}
--send-csr ${caucase-updater-csr:csr} > ${:csr_work_path} && \
cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path}
[certificate-csr_id]
recipe = plone.recipe.command
certificate = ${directory:caddy-csr_id}/certificate.pem
key = ${directory:caddy-csr_id}/key.pem
stop-on-error = True
update-command = ${:command}
command =
if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
{{ parameter_dict['openssl'] }} req -new -newkey rsa:2048 -sha256 -subj \
"/O=${kedifa-csr:organization}/OU=${kedifa-csr:organizational_unit}/CN={{ instance_parameter['ipv6-random'] }}" \
-days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
fi
[expose-csr_id-configuration]
ip = {{ instance_parameter['ipv6-random'] }}
port = 17000
key = ${certificate-csr_id:key}
certificate = ${certificate-csr_id:certificate}
error-log = ${directory:log}/expose-csr_id.log
[expose-csr_id-template]
recipe = slapos.recipe.template:jinja2
template = inline:
https://:${expose-csr_id-configuration:port}/ {
bind ${expose-csr_id-configuration:ip}
tls ${expose-csr_id-configuration:certificate} ${expose-csr_id-configuration:key}
log ${expose-csr_id-configuration:error-log}
}
rendered = ${directory:caddy-csr_id}/Caddyfile
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
depends = ${store-csr_id:command}
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['caddy'] }}
-conf ${expose-csr_id-template:rendered}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root ${directory:csr_id}
wrapper-path = ${directory:service}/expose-csr_id
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[get-csr_id-certificate]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${certificate-csr_id:certificate}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
context =
import json_module json
raw common_profile {{ parameter_dict['common_profile'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
[kedifa-config]
ip = {{ instance_parameter['ipv6-random'] }}
port = {{ instance_parameter['configuration.kedifa_port'] }}
db = ${directory:kedifa}/kedifa.sqlite
certificate = ${directory:etc-kedifa}/certificate.pem
key = ${:certificate}
ca-certificate = ${directory:etc-kedifa}/ca-certificate.pem
crl = ${directory:etc-kedifa}/crl.pem
template-csr = ${directory:etc-kedifa}/template-csr.pem
pidfile = ${directory:run}/kedifa.pid
logfile = ${directory:log}/kedifa.log
[kedifa-reloader]
recipe = slapos.cookbook:wrapper
command-line = kill -SIGHUP `cat ${kedifa-config:pidfile}`
wrapper-path = ${directory:etc-run}/kedifa-reloader
[promise-kedifa-http-reply]
<= monitor-promise-base
module = check_url_available
name = kedifa-http-reply.py
# Kedifa replies 400 on /, so use it to be sure that Kedifa replied
config-http_code = 400
config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
config-ca-cert-file = ${kedifa-config:ca-certificate}
[logrotate-entry-kedifa]
<= logrotate-entry-base
name = kedifa
log = ${kedifa-config:logfile}
rotate-num = {{ instance_parameter['configuration.rotate-num'] | int }}
[kedifa]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['kedifa'] }}
--ip ${kedifa-config:ip}
--port ${kedifa-config:port}
--db ${kedifa-config:db}
--certificate ${kedifa-config:certificate}
--ca-certificate ${kedifa-config:ca-certificate}
--crl ${kedifa-config:crl}
--pidfile ${kedifa-config:pidfile}
--logfile ${kedifa-config:logfile}
wrapper-path = ${directory:service}/kedifa
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
# Publish KeDiFa configuration for upload and download for each slave
{%- set slave_kedifa_information = {} -%}
{%- for slave in slapparameter_dict['slave-list'] -%}
{%- set slave_reference = slave['slave_reference'] -%}
{%- set slave_dict = {} -%}
{%- do slave_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%}
{%- do slave_kedifa_information.__setitem__(slave_reference, slave_dict) %}
[{{ slave_reference }}-auth-random-generate]
recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_}
command =
[ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
update-command = ${:command}
[{{ slave_reference }}-auth-random]
recipe = collective.recipe.shelloutput
file = {{ '${' + slave_reference }}-auth-random-generate:file}
commands =
passwd = cat ${:file} 2>/dev/null || echo "NotReadyYet"
{% endfor %}
[master-auth-random-generate]
recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_}
command =
[ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
update-command = ${:command}
[master-auth-random]
recipe = collective.recipe.shelloutput
file = ${master-auth-random-generate:file}
commands =
passwd = cat ${:file} 2>/dev/null || echo "NotReadyYet"
[slave-kedifa-information]
recipe = slapos.cookbook:publish.serialised
{# sort_keys are important in order to avoid shuffling parameters on each run #}
slave-kedifa-information = {{ json_module.dumps(slave_kedifa_information, sort_keys=True) }}
caucase-url = {{ caucase_url }}
master-key-generate-auth-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}/generateauth
master-key-upload-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}?auth=
master-key-download-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}
csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
csr_id-certificate = ${get-csr_id-certificate:certificate}
monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
{%- endif -%} {# if slap_software_type in software_type #}
instance-output-schema.json 0000664 0000000 0000000 00000005314 13712426147 0033333 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Caddy Frontend instanciation",
"properties": {
"accepted-slave-amount": {
"description": "Amount of Slaves allocated to the Instance which are deployed",
"type": "integer"
},
"domain": {
"description": "Base domain used by the instance",
"type": "string"
},
"master-key-generate-auth-url": {
"description": "URL to GET once auth for master-key-upload-url",
"type": "string"
},
"master-key-upload-url": {
"description": "URL to PUT PEM bundle of main certificate and key",
"type": "string"
},
"monitor-base-url": {
"description": "Base url for monitor",
"type": "string"
},
"monitor-password": {
"description": "Password to access the monitor",
"type": "string"
},
"monitor-url": {
"description": "Url of Monitor (opml)",
"type": "string"
},
"monitor-user": {
"description": "User to access the monitor",
"type": "string"
},
"rejected-slave-amount": {
"description": "Rejected Amount of Slaves allocated to the Instance which are not deployed",
"type": "integer"
},
"rejected-slave-dict": {
"description": "Dict of slaves which were rejected. Keys are slave references, values are lists of errors on slaves.",
"type": "array"
},
"slave-amount": {
"description": "Total amount of Slaves allocated to the Instance (include blocked ones)",
"type": "integer"
},
"kedifa-csr_id-url": {
"description": "URL on which KeDiFa publishes its csr_id sent to caucase.",
"type": "string"
},
"kedifa-csr_id-certificate": {
"description": "Certificate used to serve data on kedifa-csr_id-url.",
"type": "string"
},
"kedifa-caucase-url": {
"description": "Url to caucase used by KeDiFa.",
"type": "string"
},
"caddy-frontend-N-csr_id-url": {
"description": "URL on which frontend node number N publishes its csr_id sent to caucase.",
"type": "string"
},
"caddy-frontend-N-csr_id-certificate": {
"description": "Certificate used to serve data on caddy-frontend-N-csr_id-url.",
"type": "string"
},
"warning-slave-dict": {
"description": "Dict of slaves which got warnings. Keys are slave references, values are lists of warnings on slaves.",
"type": "array"
},
"warning-list": {
"description": "List of warning found during the request.",
"type": "array"
},
"backend-client-caucase-url": {
"description": "URL to caucase used by authentication to the backend.",
"type": "string"
}
},
"type": "object"
}
instance-slave-caddy-input-schema.json 0000664 0000000 0000000 00000020314 13712426147 0035321 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"custom_domain": {
"description": "Custom Domain to use for the website",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "Custom Domain",
"type": "string"
},
"url": {
"description": "Url of the backend",
"pattern": "^(http|https|ftp)://",
"title": "Backend URL",
"type": "string"
},
"type": {
"default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
"enum": [
"",
"zope",
"redirect",
"notebook",
"websocket",
"eventsource"
],
"title": "Backend Type",
"type": "string"
},
"path": {
"default": "",
"description": "Path to proxy to in the backend",
"title": "type:zope Backend Path",
"type": "string"
},
"enable_cache": {
"default": "false",
"description": "If set to true, http caching server (Apache Traffic Server) will be used between frontend Caddy and backend",
"enum": [
"false",
"true"
],
"title": "Enable Cache",
"type": "string"
},
"https-only": {
"default": "true",
"description": "If set to true, http requests will be redirected to https",
"enum": [
"false",
"true"
],
"title": "HTTPS Only",
"type": "string"
},
"default-path": {
"default": "",
"description": "Provide default path to redirect user to when user access / (the site root)",
"title": "type:zope Default Path",
"type": "string"
},
"disable-no-cache-request": {
"default": "false",
"description": "If set to true, Cache-Control and Pragma requests headers will not be sent to cache and backend servers. This prevents clients from bypassing cache when enable_cache is true",
"enum": [
"false",
"true"
],
"title": "enable_cache: Disable 'no-cache' requests",
"type": "string"
},
"disable-via-header": {
"default": "false",
"description": "If set to true, Via response headers will not be sent to client",
"enum": [
"false",
"true"
],
"title": "enable_cache: Disable 'Via' headers from cache",
"type": "string"
},
"disabled-cookie-list": {
"default": "",
"description": "List of Cookies separated by space that will not be sent to cache and backend servers. This is especially useful to discard analytics tracking cookies when using Vary: Cookie cache headers",
"title": "Disabled Cookies",
"type": "string"
},
"enable-http2": {
"default": "true",
"description": "Use HTTP2 Protocol for the site",
"enum": [
"true",
"false"
],
"title": "Enable HTTP2 Protocol",
"type": "string"
},
"https-url": {
"description": "HTTPS Url of the backend if it is diferent from url parameter",
"pattern": "^(http|https|ftp)://",
"title": "HTTPS Backend URL",
"type": "string"
},
"monitor-ipv4-test": {
"default": "",
"description": "IPv4 Address for the frontend keep monitoring with ping",
"title": "IPv4 Address to Monitor Packet Lost",
"type": "string"
},
"monitor-ipv6-test": {
"default": "",
"description": "IPv6 Address for the frontend keep monitoring with ping6 (without brackets)",
"title": "IPv6 Address to Monitor Packet Lost",
"type": "string"
},
"websocket-path-list": {
"default": "",
"description": "Space separated list of path to the websocket application. If not set the whole slave will be websocket, if set then / will be HTTP, and / will be WSS. In order to have ' ' in the space use '%20'",
"title": "type:websocket Websocket Application Path List",
"type": "string"
},
"websocket-transparent": {
"default": "true",
"description": "If set to false, websocket slave will be without Caddy's transparent proxy mode. Depending on the application the setting shall be false or true. Defaults to true for transparent proxying.",
"enum": [
"false",
"true"
],
"title": "type:websocket Transparent proxy",
"type": "string"
},
"prefer-gzip-encoding-to-backend": {
"default": "false",
"description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
"enum": [
"false",
"true"
],
"title": "Prefer gzip Encoding for Backend",
"type": "string"
},
"server-alias": {
"default": "",
"description": "Server Alias List separated by space",
"title": "Server Alias",
"type": "string"
},
"ssl-proxy-verify": {
"default": "false",
"description": "If set to true, Backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
"enum": [
"false",
"true"
],
"title": "Verify Backend Certificates",
"type": "string"
},
"ssl_crt": {
"default": "",
"description": "Content of the SSL Certificate file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate",
"type": "string"
},
"ssl_key": {
"default": "",
"description": "Content of the SSL Key file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Key",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file. Deprecated, please use key-upload-url.",
"textarea": true,
"title": "[DEPRECATED] SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_proxy_ca_crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
"textarea": true,
"title": "SSL Backend Authority's Certificate",
"type": "string"
},
"virtualhostroot-http-port": {
"default": 80,
"description": "Port where http requests to frontend will be redirected.",
"title": "type:zope virtualhostroot-http-port",
"type": "integer"
},
"virtualhostroot-https-port": {
"default": 443,
"description": "Port where https requests to frontend will be redirected.",
"title": "type:zope virtualhostroot-https-port",
"type": "integer"
},
"backend-connect-timeout": {
"description": "Time in seconds for establishing connection to the backend.",
"title": "Timeout for backend connection (seconds)",
"type": "integer"
},
"backend-connect-retries": {
"description": "Amount of retries to connect to the backend. The amount of backend-connect-timeout*backend-connect-retries seconds will be spent to connect to the backend.",
"title": "Amount of retries to connect to the backend.",
"type": "integer"
},
"request-timeout": {
"description": "Timeout for HTTP requests.",
"title": "HTTP Request timeout in seconds",
"type": "integer"
},
"ciphers": {
"description": "List of ciphers. Empty defaults to cluster list of ciphers, which by default are Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
"title": "Ordered space separated list of ciphers",
"type": "string"
},
"authenticate-to-backend": {
"description": "If set to true the frontend certificate will be used as authentication certificate to the backend. Note: backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
"enum": [
"false",
"true"
],
"title": "Authenticate to backend",
"type": "string"
}
},
"title": "Input Parameters",
"type": "object"
}
instance-slave-caddy-simplified-input-schema.json 0000664 0000000 0000000 00000006677 13712426147 0037464 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {
"$schema": "http://json-schema.org/draft-04/schema",
"properties": {
"url": {
"description": "Url of the backend",
"pattern": "^(http|https|ftp)://",
"title": "Backend URL",
"type": "string"
},
"type": {
"default": "",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
"enum": [
"",
"zope"
],
"title": "Backend Type",
"type": "string"
},
"custom_domain": {
"description": "Custom Domain to use for the website",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
"title": "Custom Domain",
"type": "string"
},
"server-alias": {
"default": "",
"description": "Server Alias List separated by space",
"title": "Server Alias",
"type": "string"
},
"default-path": {
"default": "",
"description": "Provide default path to redirect user to when user access / (the site root)",
"title": "type:zope Default Path",
"type": "string"
},
"disable-no-cache-request": {
"default": "false",
"description": "If set to true, Cache-Control and Pragma requests headers will not be sent to cache and backend servers. This prevents clients from bypassing cache when enable_cache is true",
"enum": [
"false",
"true"
],
"title": "enable_cache: Disable 'no-cache' requests",
"type": "string"
},
"disable-via-header": {
"default": "false",
"description": "If set to true, Via response headers will not be sent to client",
"enum": [
"false",
"true"
],
"title": "enable_cache: Disable 'Via' headers from cache",
"type": "string"
},
"disabled-cookie-list": {
"default": "",
"description": "List of Cookies separated by space that will not be sent to cache and backend servers. This is especially useful to discard analytics tracking cookies when using Vary: Cookie cache headers",
"title": "Disabled Cookies",
"type": "string"
},
"enable_cache": {
"default": "false",
"description": "If set to true, http caching server (Apache Traffic Server) will be used between frontend Caddy and backend",
"enum": [
"false",
"true"
],
"title": "Enable Cache",
"type": "string"
},
"https-only": {
"default": "true",
"description": "If set to true, http requests will be redirected to https",
"enum": [
"false",
"true"
],
"title": "HTTPS Only",
"type": "string"
},
"path": {
"default": "",
"description": "Path to proxy to in the backend",
"title": "type:zope Backend Path",
"type": "string"
},
"prefer-gzip-encoding-to-backend": {
"default": "false",
"description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
"enum": [
"false",
"true"
],
"title": "Prefer gzip Encoding for Backend",
"type": "string"
}
},
"title": "Input Parameters",
"type": "object"
}
instance-slave-output-schema.json 0000664 0000000 0000000 00000003257 13712426147 0034447 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Caddy Frontend instanciation",
"properties": {
"domain": {
"description": "Base domain used by the instance",
"type": "string"
},
"key-generate-auth-url": {
"description": "URL to GET once auth for key-upload-url",
"type": "string"
},
"key-upload-url": {
"description": "URL to PUT PEM bundle of certificate and key",
"type": "string"
},
"log-access-url": {
"description": "List of URLs to access logs",
"type": "array"
},
"public-ipv4": {
"description": "Public IPv4 to be included on DNS",
"type": "string"
},
"replication_number": {
"description": "Number of nodes the slave is replicated",
"type": "integer"
},
"secure_access": {
"description": "URL for HTTP access",
"type": "string"
},
"site_url": {
"description": "URL for HTTP access",
"type": "string"
},
"url": {
"description": "Default URL provided",
"type": "string"
},
"request-error-list": {
"description": "In case if slave has been rejected by master or has error in the request, the list contains information about each problem",
"type": "array"
},
"warning-list": {
"description": "List of warning found during the request.",
"type": "array"
},
"kedifa-caucase-url": {
"description": "URL to caucase used by KeDiFa",
"type": "string"
},
"backend-client-caucase-url": {
"description": "URL to caucase used by authentication to the backend.",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/instance.cfg.in 0000664 0000000 0000000 00000011142 13712426147 0031005 0 ustar 00root root 0000000 0000000 [buildout]
extends = {{ common_profile }}
parts =
dynamic-template-caddy-replicate
switch-softwaretype
[caddyprofiledeps]
recipe = caddyprofiledeps
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
key slap_software_type instance-parameter:slap-software-type
key slapparameter_dict instance-parameter:configuration
key slave_instance_list instance-parameter:slave-instance-list
section instance_parameter instance-parameter
${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-template-caddy-replicate:rendered}
RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
custom-personal = ${dynamic-template-caddy-replicate:rendered}
single-default = ${dynamic-template-caddy-frontend:rendered}
single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
replicate = ${dynamic-template-caddy-replicate:rendered}
kedifa = ${dynamic-template-kedifa:rendered}
[dynamic-template-caddy-frontend-parameters]
{% for key,value in template_frontend_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
[dynamic-template-caddy-frontend]
< = jinja2-template-base
template = {{ template_caddy_frontend }}
filename = instance-caddy-frontend.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-caddy-frontend-parameters
raw software_type single-custom-personal
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[dynamic-template-caddy-replicate]
< = jinja2-template-base
depends = ${caddyprofiledeps:recipe}
template = {{ template_caddy_replicate }}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
extra-context =
import subprocess_module subprocess
import functools_module functools
import validators validators
key cluster_identification instance-parameter:root-instance-title
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor {{ monitor2_template }}
raw common_profile {{ common_profile }}
section parameter_dict dynamic-template-caddy-frontend-parameters
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[dynamic-template-kedifa]
< = jinja2-template-base
template = {{ template_kedifa }}
filename = instance-kedifa.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-caddy-frontend-parameters
raw software_type kedifa
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# Define default parameter(s) that will be used later, in case user didn't
# specify it
# All parameters are available through the configuration.XX syntax.
# All possible parameters should have a default.
configuration.domain = example.org
configuration.public-ipv4 =
configuration.port = 4443
configuration.plain_http_port = 8080
configuration.plain_nginx_port = 8081
configuration.nginx_port = 9443
configuration.kedifa_port = 7879
# Warning: Caucase takes also cacuase_port+1
configuration.caucase_port = 8890
configuration.caucase_backend_client_port = 8990
configuration.apache-key =
configuration.apache-certificate =
configuration.open-port = 80 443
configuration.disk-cache-size = 8G
configuration.ram-cache-size = 1G
configuration.trafficserver-synthetic-port = 8083
configuration.trafficserver-mgmt-port = 8084
configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
configuration.enable-http2-by-default = true
configuration.global-disable-http2 = false
configuration.ciphers =
configuration.request-timeout = 600
configuration.mpm-graceful-shutdown-timeout = 5
configuration.frontend-name =
configuration.backend-connect-timeout = 5
configuration.backend-connect-retries = 3
configuration.backend-haproxy-http-port = 21080
configuration.backend-haproxy-https-port = 21443
configuration.authenticate-to-backend = False
configuration.rotate-num = 4000
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/setup.py 0000664 0000000 0000000 00000000554 13712426147 0027632 0 ustar 00root root 0000000 0000000 # The caddyprofiledeps egg allows to set dependecies of the Caddy profiles
# which are enabled during the instance run, thanks to using caddyprofiledeps
# recipe
from setuptools import setup
setup(
name='caddyprofiledeps',
install_requires=[
'validators',
],
entry_points={
'zc.buildout': [
'default = caddyprofiledummy:Recipe',
]
}
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/software.cfg 0000664 0000000 0000000 00000001134 13712426147 0030426 0 ustar 00root root 0000000 0000000 [buildout]
extends = common.cfg
[versions]
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
# Versions pinned for kedifa need urllib3 >= 1.18
urllib3 = 1.24
requests = 2.20.0
certifi = 2018.10.15
idna = 2.7
chardet = 3.0.4
# ipaddress is patching IPAddress so IPv6 match works
ipaddress = 1.0.22
# Versions pinned for kedifa need urllib3 >= 1.18
validators = 0.12.2
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
ecdsa = 0.13
gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
websockify = 0.8.0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/software.cfg.json 0000664 0000000 0000000 00000002655 13712426147 0031407 0 ustar 00root root 0000000 0000000 {
"description": "Caddy Frontend",
"name": "Caddy Frontend",
"serialisation": "xml",
"software-type": {
"custom-personal": {
"description": "",
"index": 2,
"request": "instance-caddy-input-schema.json",
"response": "instance-output-schema.json",
"title": "Custom Personal"
},
"custom-personal-slave": {
"description": "Custom Personal",
"index": 1,
"request": "instance-slave-caddy-input-schema.json",
"response": "instance-output-schema.json",
"shared": true,
"software-type": "custom-personal",
"title": "Custom Personal (Slave)"
},
"default": {
"description": "Default",
"index": 3,
"request": "instance-caddy-input-schema.json",
"response": "instance-output-schema.json",
"software-type": "default",
"title": "Default"
},
"default-slave": {
"description": "Default",
"index": 4,
"request": "instance-slave-caddy-input-schema.json",
"response": "instance-output-schema.json",
"shared": true,
"software-type": "default",
"title": "Default (slave)"
},
"default-simplified-slave": {
"description": "Default",
"index": 5,
"request": "instance-slave-caddy-simplified-input-schema.json",
"response": "instance-output-schema.json",
"shared": true,
"software-type": "default",
"title": "Default Basic (slave)"
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates/ 0000775 0000000 0000000 00000000000 13712426147 0030112 5 ustar 00root root 0000000 0000000 Caddyfile.in 0000664 0000000 0000000 00000002706 13712426147 0032254 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates # Main caddy configuration file
import {{frontend_configuration.get('log-access-configuration')}}
import {{ slave_configuration_directory }}/*.conf
:{{ https_port }} {
tls {{ master_certificate }} {{ master_certificate }}
bind {{ local_ipv4 }}
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
:{{ http_port }} {
bind {{ local_ipv4 }}
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
# Access to server-status Caddy-style
https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
bind {{ local_ipv4 }}
basicauth "{{ username }}" {{ password | trim }} {
"Server Status"
/
}
expvar
pprof
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
apache-custom-slave-list.cfg.in 0000664 0000000 0000000 00000055203 13712426147 0035740 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates {%- if software_type == slap_software_type %}
{%- set kedifa_updater_mapping = [] %}
{%- set cached_server_dict = {} %}
{%- set backend_slave_list = [] %}
{%- set part_list = [] %}
{%- set cache_port = caddy_configuration.get('cache-port') %}
{%- set cache_access = "http://%s:%s" % (local_ipv4, cache_port) %}
{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) %}
{%- set backend_haproxy_http_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['http-port']) %}
{%- set backend_haproxy_https_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['https-port']) %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': local_ipv4, 'http_port': http_port, 'https_port': https_port} %}
{%- set slave_log_dict = {} %}
{%- if extra_slave_instance_list %}
{%- set slave_instance_information_list = [] %}
{%- set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) %}
{%- endif %}
{%- if master_key_download_url %}
{%- do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
{%- else %}
{%- do kedifa_updater_mapping.append(('notreadyyet', master_certificate, apache_certificate)) %}
{%- endif %}
{%- if kedifa_configuration['slave_kedifa_information'] %}
{%- set slave_kedifa_information = json_module.loads(kedifa_configuration['slave_kedifa_information']) %}
{%- else %}
{%- set slave_kedifa_information = {} %}
{%- endif -%}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
raw common_profile {{ common_profile }}
${:extra-context}
# empty sections if no slaves are available
[slave-log-directory-dict]
[slave-password]
{#- Loop thought slave list to set up slaves #}
{%- set DEFAULT_PORT = {'http': 80, 'https': 443, '': None} %}
{%- for slave_instance in slave_instance_list %}
{#- prepare backend parameters #}
{%- for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment} %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{#- Manage ciphers #}
{%- set slave_ciphers = slave_instance.get('ciphers', '').strip().split() %}
{%- if slave_ciphers %}
{%- set slave_cipher_list = ' '.join(slave_ciphers) %}
{%- else %}
{%- set slave_cipher_list = ciphers.strip() %}
{%- endif %}
{%- do slave_instance.__setitem__('cipher_list', slave_cipher_list) %}
{#- Manage common instance parameters #}
{%- set slave_type = slave_instance.get('type', '') %}
{%- set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %}
{%- set slave_reference = slave_instance.get('slave_reference') %}
{%- set slave_kedifa = slave_kedifa_information.get(slave_reference) %}
{#- Setup backend URLs for front facing Caddy #}
{%- if slave_type == 'redirect' %}
{%- do slave_instance.__setitem__('backend-http-url', slave_instance.get('url', '').rstrip('/')) %}
{%- if slave_instance.get('https-url') %}
{%- do slave_instance.__setitem__('backend-https-url', slave_instance.get['https-url'].rstrip('/')) %}
{%- endif %}
{%- elif enable_cache %}
{%- if 'domain' in slave_instance %}
{%- if not slave_instance.get('custom_domain') %}
{%- do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) %}
{%- endif %}
{%- endif %}
{%- do slave_instance.__setitem__('backend-http-url', cache_access) %}
{%- if slave_instance.get('https-url') %}
{%- do slave_instance.__setitem__('backend-https-url', ssl_cache_access) %}
{%- endif %}
{%- do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) %}
{%- else %}
{%- do slave_instance.__setitem__('backend-http-url', backend_haproxy_http_url) %}
{%- if slave_instance.get('https-url') %}
{%- do slave_instance.__setitem__('backend-https-url', backend_haproxy_https_url) %}
{%- endif %}
{%- endif %}
{%- if slave_kedifa %}
{%- set key_download_url = slave_kedifa.get('key-download-url') %}
{%- else %}
{%- set key_download_url = 'notreadyyet' %}
{%- endif %}
{%- set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference %}
{%- set slave_parameter_dict = generic_instance_parameter_dict.copy() %}
{%- set slave_publish_dict = {} %}
{%- set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
{%- set slave_logrotate_section = slave_reference + "-logs" %}
{%- set slave_password_section = slave_reference + "-password" %}
{%- set slave_ln_section = slave_reference + "-ln" %}
{#- extend parts #}
{%- do part_list.extend([slave_ln_section]) %}
{%- do part_list.extend([slave_logrotate_section, slave_section_title]) %}
{%- set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
{#- Pass HTTP2 switch #}
{%- do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
{%- do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
{#- Pass backend timeout values #}
{%- for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
{%- if slave_instance.get(key, '') == '' %}
{%- do slave_instance.__setitem__(key, configuration[key]) %}
{%- endif %}
{%- endfor %}
{%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
{#- Set Up log files #}
{%- do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
{%- do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
{%- do slave_parameter_dict.__setitem__('backend_log', '/'.join([caddy_log_directory, '%s_backend_log' % slave_reference])) %}
{%- do slave_instance.__setitem__('access_log', slave_parameter_dict.get('access_log')) %}
{%- do slave_instance.__setitem__('error_log', slave_parameter_dict.get('error_log')) %}
{%- do slave_instance.__setitem__('backend_log', slave_parameter_dict.get('backend_log')) %}
{#- Add slave log directory to the slave log access dict #}
{%- do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
{%- set slave_log_access_url = 'https://' + slave_reference.lower() + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('caddy-ipv6') + ']:' + frontend_configuration.get('caddy-https-port') + '/' + slave_reference.lower() + '/' %}
{%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
{%- do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
{%- do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
{%- do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %}
{#- Set slave domain if none was defined #}
{%- if slave_instance.get('custom_domain', None) == None %}
{%- set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() %}
{%- do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) %}
{%- endif %}
{%- do slave_publish_dict.__setitem__('domain', slave_instance.get('custom_domain')) %}
{%- do slave_publish_dict.__setitem__('url', "http://%s" % slave_instance.get('custom_domain')) %}
{%- do slave_publish_dict.__setitem__('site_url', "http://%s" % slave_instance.get('custom_domain')) %}
{%- do slave_publish_dict.__setitem__('secure_access', 'https://%s' % slave_instance.get('custom_domain')) %}
[slave-log-directory-dict]
{{slave_reference}} = {{ slave_log_folder }}
[slave-password]
{{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
{#- Set slave logrotate entry #}
[{{slave_logrotate_section}}]
<= logrotate-entry-base
name = ${:_buildout_section_name_}
log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}} {{slave_parameter_dict.get('backend_log')}}
backup = {{ slave_log_folder }}
rotate-num = {{ dumps('' ~ configuration['rotate-num']) }}
{#- integrate current logs inside #}
[{{slave_ln_section}}]
recipe = plone.recipe.command
stop-on-error = false
update-command = ${:command}
command = ln -sf {{slave_parameter_dict.get('error_log')}} {{ slave_log_folder }}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} {{ slave_log_folder }}/access.log && ln -sf {{slave_parameter_dict.get('backend_log')}} {{ slave_log_folder }}/backend.log
{#- Set password for slave #}
[{{slave_password_section}}]
recipe = slapos.cookbook:generate.password
storage-path = {{caddy_configuration_directory}}/.{{slave_reference}}.passwd
bytes = 8
{#- ################################################## #}
{#- Set Slave Certificates if needed #}
{#- Set certificate key for custom configuration #}
{%- set cert_name = slave_reference.replace('-','.') + '.pem' %}
{%- set certificate = '%s/%s' % (autocert, cert_name) %}
{%- do slave_parameter_dict.__setitem__('certificate', certificate )%}
{#- Set ssl certificates for each slave #}
{%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
{%- if cert_name in slave_instance %}
{%- set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}
{%- set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
{%- do part_list.append(cert_title) %}
{%- do slave_parameter_dict.__setitem__(cert_name, cert_file) %}
{%- do slave_instance.__setitem__('path_to_' + cert_name, cert_file) %}
{#- Store certificates on fs #}
[{{ cert_title }}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
extra-context =
key content {{ cert_title + '-config:value' }}
{#- BBB: SlapOS Master non-zero knowledge BEGIN #}
{#- Store certificate in config #}
[{{ cert_title + '-config' }}]
value = {{ dumps(slave_instance.get(cert_name)) }}
{%- endif %}
{%- endfor %}
{#- Set Up Certs #}
{%- if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
{%- set cert_title = '%s-crt' % (slave_reference) %}
{%- set cert_file = '/'.join([directory['bbb-ssl-dir'], cert_title.replace('-','.')]) %}
{%- do kedifa_updater_mapping.append((key_download_url, certificate, cert_file)) %}
{%- do part_list.append(cert_title) %}
{%- do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %}
[{{cert_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '') + '\n' + slave_instance.get('ssl_key')) }}
extra-context =
key content :cert-content
{%- else %}
{%- do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %}
{%- endif %}
{#- BBB: SlapOS Master non-zero knowledge END #}
{#- ########################################## #}
{#- Set Slave Configuration #}
[{{ slave_configuration_section_name }}]
certificate = {{ certificate }}
https_port = {{ dumps('' ~ https_port) }}
http_port = {{ dumps('' ~ http_port) }}
local_ipv4 = {{ dumps('' ~ local_ipv4) }}
{%- for key, value in slave_instance.iteritems() %}
{%- if value is not none %}
{{ key }} = {{ dumps('' ~ value) }}
{%- endif %}
{%- endfor %}
[{{ slave_section_title }}]
< = jinja2-template-base
rendered = {{ caddy_configuration_directory }}/${:filename}
template = {{ template_default_slave_configuration }}
extra-context =
section slave_parameter {{ slave_configuration_section_name }}
import urllib_module urllib
filename = {{ '%s.conf' % slave_reference }}
{{ '\n' }}
{%- set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %}
{%- if monitor_ipv6_test %}
{%- set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %}
{%- do part_list.append(monitor_ipv6_section_title) %}
[{{ monitor_ipv6_section_title }}]
<= monitor-promise-base
module = check_icmp_packet_lost
name = {{ monitor_ipv6_section_title }}.py
config-address = {{ dumps(monitor_ipv6_test) }}
# promise frequency in minutes (2 times/day)
config-frequency = 720
{%- endif %}
{%- set monitor_ipv4_test = slave_instance.get('monitor-ipv4-test', '') %}
{%- if monitor_ipv4_test %}
{%- set monitor_ipv4_section_title = 'check-%s-ipv4-packet-list-test' % slave_instance.get('slave_reference') %}
{%- do part_list.append(monitor_ipv4_section_title) %}
[{{ monitor_ipv4_section_title }}]
<= monitor-promise-base
module = check_icmp_packet_lost
name = {{ monitor_ipv4_section_title }}.py
config-address = {{ dumps(monitor_ipv4_test) }}
config-ipv4 = true
# promise frequency in minutes (2 times/day)
config-frequency = 720
{%- endif %}
{#- ############################### #}
{#- Publish Slave Information #}
{%- if not extra_slave_instance_list %}
{%- set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') %}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
{%- for key, value in slave_publish_dict.iteritems() %}
{{ key }} = {{ value }}
{%- endfor %}
{%- else %}
{%- do slave_instance_information_list.append(slave_publish_dict) %}
{%- endif %}
{%- if slave_type != 'redirect' %}
{%- do backend_slave_list.append(slave_instance) %}
{%- endif %}
{%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #}
[slave-log-directories]
<= slave-log-directory-dict
recipe = slapos.cookbook:mkdirectory
{%- do part_list.append('slave-log-directories') %}
{%- do part_list.append('caddy-log-access') %}
{#- ############################################## #}
{#- ## Prepare virtualhost for slaves using cache #}
{#- Define IPv6 to IPV4 tunneling #}
[tunnel-6to4-base]
recipe = slapos.cookbook:wrapper
ipv4 = ${slap-network-information:local-ipv4}
ipv6 = ${slap-network-information:global-ipv6}
wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
command-line = {{ sixtunnel_executable }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[tunnel-6to4-base-http_port]
<= tunnel-6to4-base
ipv4-port = {{ http_port }}
ipv6-port = {{ http_port }}
[tunnel-6to4-base-https_port]
<= tunnel-6to4-base
ipv4-port = {{ https_port }}
ipv6-port = {{ https_port }}
{#- Define log access #}
[caddy-log-access-parameters]
caddy_log_directory = {{ dumps(caddy_log_directory) }}
caddy_configuration_directory = {{ dumps(caddy_configuration_directory) }}
local_ipv4 = {{ dumps(local_ipv4) }}
global_ipv6 = {{ dumps(global_ipv6) }}
https_port = {{ dumps(https_port) }}
http_port = {{ dumps(http_port) }}
ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }}
access_log = {{ dumps(access_log) }}
error_log = {{ dumps(error_log) }}
not_found_file = {{ dumps(not_found_file) }}
[caddy-log-access]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directory-dict
section slave_password slave-password
section parameter_dict caddy-log-access-parameters
{#- Publish information for the instance #}
[publish-caddy-information]
recipe = slapos.cookbook:publish.serialised
public-ipv4 = {{ public_ipv4 }}
private-ipv4 = {{ local_ipv4 }}
{%- if extra_slave_instance_list %}
{#- sort_keys are important in order to avoid shuffling parameters on each run #}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list, sort_keys=True) }}
{%- endif %}
monitor-base-url = {{ monitor_base_url }}
csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
backend-client-csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/backend-haproxy-csr_id.txt
csr_id-certificate = ${get-csr_id-certificate:certificate}
[kedifa-updater]
recipe = slapos.cookbook:wrapper
command-line = {{ kedifa_configuration['kedifa-updater'] }}
--server-ca-certificate {{ kedifa_configuration['ca-certificate'] }}
--identity {{ kedifa_configuration['certificate'] }}
--master-certificate {{ master_certificate }}
--on-update "{{ frontend_graceful_reload }}"
${kedifa-updater-mapping:file}
{{ kedifa_configuration['kedifa-updater-state-file'] }}
wrapper-path = {{ directory['service'] }}/kedifa-updater
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[kedifa-updater-run]
recipe = plone.recipe.command
stop-on-error = True
command = {{ kedifa_configuration['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ frontend_graceful_reload }}"
update-command = ${:command}
[kedifa-updater-mapping]
recipe = slapos.recipe.template:jinja2
file = {{ kedifa_configuration['kedifa-updater-mapping-file'] }}
template = inline:
{%- for mapping in kedifa_updater_mapping %}
{{ mapping[0] }} {{ mapping[1] }} {{ mapping[2] }}
{%- endfor %}
rendered = ${:file}
[caddy-log-access-empty]
# Caddy refuse to start if an `import`ed file is empty, so we prepend a header
# so that the file is never empty.
< = jinja2-template-base
template = inline: # This file contain directives to serve directories with log files for shared instances, but no shared instances are defined yet.
rendered = {{frontend_configuration.get('log-access-configuration')}}
##
[backend-haproxy-configuration]
< = jinja2-template-base
template = {{ template_backend_haproxy_configuration }}
rendered = ${backend-haproxy-config:file}
backend_slave_list = {{ dumps(sorted(backend_slave_list)) }}
extra-context =
key backend_slave_list :backend_slave_list
section configuration backend-haproxy-config
[backend-haproxy-config]
file = {{ backend_haproxy_configuration['file'] }}
pid-file = {{ backend_haproxy_configuration['pid-file'] }}
log-socket = {{ backend_haproxy_configuration['log-socket'] }}
local-ipv4 = {{ dumps('' ~ local_ipv4) }}
http-port = {{ ('' ~ backend_haproxy_configuration['http-port']) }}
https-port = {{ ('' ~ backend_haproxy_configuration['https-port']) }}
request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }}
certificate = {{ dumps('' ~ backend_haproxy_configuration['certificate']) }}
[store-backend-haproxy-csr_id]
recipe = plone.recipe.command
csr_id_path = {{ directory['csr_id'] }}/backend-haproxy-csr_id.txt
csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ bin_directory }}/caucase \
--ca-url {{ backend_haproxy_configuration['caucase-url'] }} \
--ca-crt {{ backend_haproxy_configuration['cas-ca-certificate'] }} \
--crl {{ backend_haproxy_configuration['crl'] }} \
--mode service \
--send-csr {{ backend_haproxy_configuration['csr'] }} > ${:csr_work_path} && \
cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path}
##
[buildout]
extends =
{{ common_profile }}
{{ logrotate_base_instance }}
{{ monitor_template }}
parts +=
kedifa-updater
kedifa-updater-run
backend-haproxy-configuration
{%- for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
{%- if 'caddy-log-access' not in part_list %}
caddy-log-access-empty
{%- endif %}
publish-caddy-information
tunnel-6to4-base-http_port
tunnel-6to4-base-https_port
expose-csr_id
promise-expose-csr_id-ip-port
cache-access = {{ cache_access }}
[store-csr_id]
recipe = plone.recipe.command
csr_id_path = {{ directory['csr_id'] }}/csr_id.txt
csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ bin_directory }}/caucase \
--ca-url {{ kedifa_configuration['caucase-url'] }} \
--ca-crt {{ kedifa_configuration['cas-ca-certificate'] }} \
--crl {{ kedifa_configuration['crl'] }} \
--mode service \
--send-csr {{ kedifa_configuration['csr'] }} > ${:csr_work_path} && \
cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path}
[certificate-csr_id]
recipe = plone.recipe.command
certificate = {{ directory['caddy-csr_id'] }}/certificate.pem
key = {{ directory['caddy-csr_id'] }}/key.pem
stop-on-error = True
update-command = ${:command}
command =
if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
openssl req -new -newkey rsa:2048 -sha256 -subj \
"/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-network-information:global-ipv6}" \
-days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
fi
[expose-csr_id-configuration]
ip = ${slap-network-information:global-ipv6}
port = 17001
key = ${certificate-csr_id:key}
certificate = ${certificate-csr_id:certificate}
error-log = {{ directory['caddy-csr_id-log'] }}/expose-csr_id.log
[expose-csr_id-template]
recipe = slapos.recipe.template:jinja2
template = inline:
https://:${expose-csr_id-configuration:port}/ {
bind ${expose-csr_id-configuration:ip}
tls ${expose-csr_id-configuration:certificate} ${expose-csr_id-configuration:key}
log ${expose-csr_id-configuration:error-log}
}
rendered = {{ directory['caddy-csr_id'] }}/Caddyfile
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
depends =
${store-csr_id:command}
${store-backend-haproxy-csr_id:command}
recipe = slapos.cookbook:wrapper
command-line = {{ caddy_executable }}
-conf ${expose-csr_id-template:rendered}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root {{ directory['csr_id'] }}
wrapper-path = {{ directory['service'] }}/expose-csr_id
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[get-csr_id-certificate]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${certificate-csr_id:certificate}
{%- endif %} {# if software_type == slap_software_type #}
apache-lazy-script-call.sh.in 0000664 0000000 0000000 00000000566 13712426147 0035414 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!/bin/bash
PID=$$
PIDFILE={{ pid_file }}
sleep $((RANDOM % 10))
if [ ! -f $PIDFILE ]; then
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
rm -f $PIDFILE
else
ps --pid `cat $PIDFILE` &>/dev/null
if [ $? -eq 0 ]; then
echo "Skipped"
else
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
rm -f $PIDFILE
fi
fi
backend-haproxy-rsyslogd.conf.in 0000664 0000000 0000000 00000002156 13712426147 0036236 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates module(
load="imuxsock"
SysSock.Name="{{ configuration['log-socket'] }}")
# Just simply output the raw line without any additional information, as
# haproxy emits enough information by itself
# Also cut out first empty space in msg, which is related to rsyslogd
# internal and end up cutting on 8k, as it's default of $MaxMessageSize
template(name="rawoutput" type="string" string="%msg:2:8192%\n")
$ActionFileDefaultTemplate rawoutput
$FileCreateMode 0600
$DirCreateMode 0700
$Umask 0022
$WorkDirectory {{ configuration['spool-directory'] }}
# Setup logging per slave, by extracting the slave name from the log stream
{%- set regex = ".*-backend (.*)-http.*" %}
template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
set $!slave_name = exec_template("extract_slave_name");
template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")
if (re_match($msg, "{{ regex }}")) then {
action(type="omfile" dynaFile="slave_output")
stop
}
{#- emit all not catched messages to full log file #}
*.* {{ configuration['log-file'] }}
backend-haproxy.cfg.in 0000664 0000000 0000000 00000006601 13712426147 0034203 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates global
pidfile {{ configuration['pid-file'] }}
# master-worker is compatible with foreground with process management
master-worker
log {{ configuration['log-socket'] }} local0
defaults
log global
mode http
option httplog
timeout queue 60s
timeout server {{ configuration['request-timeout'] }}s
timeout client {{ configuration['request-timeout'] }}s
timeout connect {{ configuration['backend-connect-timeout'] }}s
retries {{ configuration['backend-connect-retries'] }}
{%- macro frontend_entry(slave_instance, scheme) %}
{%- set host_list = (slave_instance.get('server-alias') or '').split() %}
{%- if slave_instance.get('custom_domain') not in host_list %}
{%- do host_list.append(slave_instance.get('custom_domain')) %}
{%- endif %}
{%- for host in host_list %}
{%- if host.startswith('*.') %}
{#- hdr_sub has to be used, as anyway something.example.com shall match *.example.com[:port], with optional port #}
acl is_{{ slave_instance['slave_reference'] }} hdr_sub(host) -i {{ host[2:] }}
{%- else %}
acl is_{{ slave_instance['slave_reference'] }} hdr_dom(host) -i {{ host }}
{%- endif %}
{%- endfor %}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
{%- endmacro %}
frontend http-backend
bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
{%- for slave_instance in backend_slave_list %}
{{ frontend_entry(slave_instance, 'http') }}
{%- endfor %}
frontend https-backend
bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
{%- for slave_instance in backend_slave_list %}
{{ frontend_entry(slave_instance, 'https') }}
{%- endfor %}
{%- for slave_instance in backend_slave_list %}
{%- for (scheme, prefix) in [('http', 'http_backend'), ('https', 'https_backend')] %}
{%- set info_dict = slave_instance[prefix] %}
{%- if info_dict['scheme'] == 'https' %}
{%- set ssl = [] %}
{%- if slave_instance['authenticate-to-backend'] %}
{%- set ssl = ['crt %s' % (configuration['certificate'],)] %}
{%- endif %}
{%- do ssl.append('ssl verify') %}
{%- set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
{%- if slave_instance['ssl_proxy_verify'] %}
{%- if path_to_ssl_proxy_ca_crt %}
{%- do ssl.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
{%- else %}
{#- Backend SSL shall be verified, but not CA provided, disallow connection #}
{#- Simply dropping hostname from the dict will result with ignoring it... #}
{%- do info_dict.__setitem__('hostname', '') %}
{%- endif %}
{%- else %}
{%- do ssl.append('none') %}
{%- endif %}
{%- set ssl = ' '.join(ssl) %}
{%- else %}
{%- set ssl = '' %}
{%- endif %}
backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- set hostname = info_dict['hostname'] %}
{%- set port = info_dict['port'] %}
{%- set path = info_dict['path'].rstrip('/') %}
{%- if hostname and port %}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server backend {{ hostname }}:{{ port }} {{ ssl }}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endfor %}
caddy-backend-url-validator.in 0000664 0000000 0000000 00000000235 13712426147 0035617 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!${dash:location}/bin/dash
config="https://example.com {\n proxy / $1 {\n }\n}"
echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1
configuration-state-script.sh.in 0000664 0000000 0000000 00000000745 13712426147 0036271 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!/bin/sh
set -e
SIGNATURE_FILE={{ signature_file }}
NSIGNATURE_FILE={{ signature_file }}.tmp
touch $SIGNATURE_FILE
{{ sha256sum }} {{ path_list }} 2> /dev/null | sort -k 66 > $NSIGNATURE_FILE
if diff "$SIGNATURE_FILE" "$NSIGNATURE_FILE" > /dev/null ; then
# No changes since last run just propagate information
rm -f ${NSIGNATURE_FILE}
exit 1
else
# Changes since last run, so store new value and propagate information
mv "$NSIGNATURE_FILE" "$SIGNATURE_FILE"
exit 0
fi
default-virtualhost.conf.in 0000664 0000000 0000000 00000023561 13712426147 0035324 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
{%- set enable_cache = slave_parameter.get('enable_cache', '').lower() in TRUE_VALUES %}
{%- set disable_no_cache_header = slave_parameter.get('disable-no-cache-request', '').lower() in TRUE_VALUES %}
{%- set disable_via_header = slave_parameter.get('disable-via-header', '').lower() in TRUE_VALUES %}
{%- set prefer_gzip = slave_parameter.get('prefer-gzip-encoding-to-backend', '').lower() in TRUE_VALUES %}
{%- set proxy_append_list = [('', 'Default proxy configuration')] %}
{%- if prefer_gzip %}
{%- do proxy_append_list.append(('prefer-gzip', 'Proxy which always overrides Accept-Encoding to gzip if such is found')) %}
{%- endif %} {#- if prefer_gzip #}
{%- set server_alias_list = slave_parameter.get('server-alias', '').split() %}
{%- set enable_h2 = slave_parameter['global_disable_http2'].lower() not in TRUE_VALUES and slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default']).lower() in TRUE_VALUES %}
{%- set disabled_cookie_list = slave_parameter.get('disabled-cookie-list', '').split() %}
{%- set https_only = slave_parameter.get('https-only', 'true').lower() in TRUE_VALUES %}
{%- set slave_type = slave_parameter.get('type', '') %}
{%- set host_list = server_alias_list %}
{%- set cipher_list = slave_parameter.get('cipher_list', '').strip() %}
{%- if slave_parameter.get('custom_domain') not in host_list %}
{%- do host_list.append(slave_parameter.get('custom_domain')) %}
{%- endif %}
{%- set http_host_list = [] %}
{%- set https_host_list = [] %}
{%- for host in host_list %}
{%- do http_host_list.append('http://%s:%s' % (host, slave_parameter['http_port'] )) %}
{%- do https_host_list.append('https://%s:%s' % (host, slave_parameter['https_port'] )) %}
{%- endfor %} {#- for host in host_list #}
{%- set default_path = slave_parameter.get('default-path', '').strip('/') | urlencode %}
{%- set websocket_path_list = [] %}
{%- for websocket_path in slave_parameter.get('websocket-path-list', '').split() %}
{%- set websocket_path = websocket_path.strip('/') %}
{#- Unquote the path, so %20 and similar can be represented correctly #}
{%- set websocket_path = urllib_module.unquote(websocket_path.strip()) %}
{%- if websocket_path %}
{%- do websocket_path_list.append(websocket_path) %}
{%- endif %}
{%- endfor %}
{%- set websocket_transparent = slave_parameter.get('websocket-transparent', 'true').lower() in TRUE_VALUES %}
{%- if slave_type in ['notebook', 'websocket'] %}
{# websocket style needs http 1.1 max #}
{%- set enable_h2 = False %}
{%- endif %}
{%- macro proxy_header() %}
timeout {{ slave_parameter['request-timeout'] }}s
# force reset of X-Forwarded-For
header_upstream X-Forwarded-For {remote}
{%- endmacro %} {# proxy_header #}
{%- for tls in [True, False] %}
{%- if tls %}
{%- set backend_url = slave_parameter.get('backend-https-url', slave_parameter.get('backend-http-url')) %}
# SSL enabled hosts
{{ https_host_list|join(', ') }} {
{%- else %}
{%- set backend_url = slave_parameter['backend-http-url'] %}
# SSL-disabled hosts
{{ http_host_list|join(', ') }} {
{%- endif %}
bind {{ slave_parameter['local_ipv4'] }}
{%- if tls %}
tls {{ slave_parameter['certificate'] }} {{ slave_parameter['certificate'] }} {
{%- if cipher_list %}
ciphers {{ cipher_list }}
{%- endif %}
{%- if enable_h2 %}
# Allow HTTP2
alpn h2 http/1.1
{%- else %} {#- if enable_h2 #}
# Disallow HTTP2
alpn http/1.1
{%- endif %} {#- if enable_h2 #}
} {# tls #}
{%- endif %} {#- if tls #}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log') }} {
rotate_size 0
}
{%- if not (slave_type == 'zope' and backend_url) %}
{% if prefer_gzip and not (not tls and https_only) %}
rewrite {
regexp (.*)
if {>Accept-Encoding} match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
to /prefer-gzip{1}
}
rewrite {
regexp (.*)
if {>Accept-Encoding} not_match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
to {1}
}
{% elif slave_type not in ['notebook', 'websocket'] %}
rewrite {
regexp (.*)
to {1}
}
{% endif %} {# elif slave_type != 'notebook' #}
{%- endif %} {#- if not (slave_type == 'zope' and backend_url) #}
{%- if not tls and https_only %}
# Enforced redirection to SSL-enabled host
redir 302 {
/ https://{host}{rewrite_uri}
}
{%- elif slave_type == 'zope' and backend_url %}
# Zope configuration
{%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }}
proxy "/{{ proxy_name }}" {{ backend_url }} {
{{ proxy_header() }}
{%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip
header_upstream Accept-Encoding gzip
{%- endif %} {#- if proxy_name == 'prefer-gzip' #}
{%- for disabled_cookie in disabled_cookie_list %}
# Remove cookie {{ disabled_cookie }} from client Cookies
header_upstream Cookie "(.*)(^{{ disabled_cookie }}=[^;]*; |; {{ disabled_cookie }}=[^;]*|^{{ disabled_cookie }}=[^;]*$)(.*)" "$1 $3"
{%- endfor %} {#- for disabled_cookie in disabled_cookie_list #}
{%- if disable_via_header %}
header_downstream -Via
{%- endif %} {#- if disable_via_header #}
{%- if disable_no_cache_header %}
header_upstream -Cache-Control
header_upstream -Pragma
{%- endif %} {#- if disable_no_cache_header #}
transparent
} {# proxy #}
{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #}
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if default_path #}
{%- if prefer_gzip and not (not tls and https_only) %}
rewrite {
regexp (.*)
if {>Accept-Encoding} match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
{%- if tls %}
to /prefer-gzip/VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- else %}
to /prefer-gzip/VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- endif %}
}
rewrite {
regexp (.*)
if {>Accept-Encoding} not_match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
{%- if tls %}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- else %}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- endif %}
}
{%- else %}
rewrite {
regexp (.*)
{%- if tls %}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- else %}
to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') | int }}%2F{{ slave_parameter.get('path', '').strip('/') }}%2FVirtualHostRoot/{1}
{%- endif %}
} {# rewrite #}
{%- endif %} {#- if prefer_gzip #}
{%- elif slave_type == 'redirect' %}
{%- if backend_url %}
# Redirect configuration
redir 302 {
/ {{ backend_url }}{rewrite_uri}
}
{%- endif %}
{%- elif slave_type == 'notebook' %}
proxy / {{ backend_url }} {
{{ proxy_header() }}
transparent
}
rewrite {
regexp "/(api/kernels/[^/]+/(channels|iopub|shell|stdin)|terminals/websocket)/?"
to /proxy/{1}
}
proxy /proxy/ {{ backend_url }} {
{{ proxy_header() }}
transparent
websocket
without /proxy/
}
{%- elif slave_type == 'websocket' %}
{%- if websocket_path_list %}
proxy / {{ backend_url }} {
{{ proxy_header() }}
{%- if websocket_transparent %}
transparent
{%- else %}
header_upstream Host {host}
{%- endif %}
}
{%- for websocket_path in websocket_path_list %}
proxy "/{{ websocket_path }}" {{ backend_url }} {
{{ proxy_header() }}
websocket
{%- if websocket_transparent %}
transparent
{%- else %}
header_upstream Host {host}
{%- endif %}
}
{%- endfor %}
{%- else %}
proxy / {{ backend_url }} {
{{ proxy_header() }}
websocket
{%- if websocket_transparent %}
transparent
{%- else %}
header_upstream Host {host}
{%- endif %}
}
{%- endif %}
{%- else %} {#- if slave_type == 'zope' and backend_url #}
# Default configuration
{%- if default_path %}
redir 301 {
if {path} is /
/ {scheme}://{host}/{{ default_path }}
} {# redir #}
{%- endif %} {#- if default_path #}
{%- if backend_url %}
{%- for (proxy_name, proxy_comment) in proxy_append_list %}
# {{ proxy_comment }}
proxy "/{{ proxy_name }}" {{ backend_url }} {
{{ proxy_header() }}
{%- if proxy_name == 'prefer-gzip' %}
without /prefer-gzip
header_upstream Accept-Encoding gzip
{%- endif %} {#- if proxy_name == 'prefer-gzip' #}
{%- for disabled_cookie in disabled_cookie_list %}
# Remove cookie {{ disabled_cookie }} from client Cookies
header_upstream Cookie "(.*)(^{{ disabled_cookie }}=[^;]*; |; {{ disabled_cookie }}=[^;]*|^{{ disabled_cookie }}=[^;]*$)(.*)" "$1 $3"
{%- endfor %} {#- for disabled_cookie in disabled_cookie_list #}
{%- if disable_via_header %}
header_downstream -Via
{%- endif %} {#- if disable_via_header #}
{%- if disable_no_cache_header %}
header_upstream -Cache-Control
header_upstream -Pragma
{%- endif %} {#- if disable_no_cache_header #}
transparent
} {# proxy #}
{%- endfor %} {#- for (proxy_name, proxy_comment) in proxy_append_list #}
{%- endif %} {#- if backend_url #}
{%- endif %} {#- if slave_type == 'zope' and backend_url #}
} {# https_host_list|join(', ') #}
{%- endfor %} {#- for tls in [True, False] #}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates/empty.in 0000664 0000000 0000000 00000000016 13712426147 0031575 0 ustar 00root root 0000000 0000000 {{ content }}
graceful-script.sh.in 0000664 0000000 0000000 00000000262 13712426147 0034066 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!/bin/sh
set -e
if {{ caddy_configuration_state }} ; then
echo "Reloading.."
{{ graceful_reload_command }}
else
echo "Nothing changed, so nothing to reload"
exit 0
fi
nginx-eventsource-slave.conf.in 0000664 0000000 0000000 00000010771 13712426147 0036110 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates {% set url = slave_parameter.get('url') %}
{% set https_url = slave_parameter.get('https-url', url) %}
{% if url.startswith("http://") or url.startswith("https://") %}
{% set upstream = url.split("/")[2] %}
{% set https_upstream = https_url.split("/")[2] %}
{% set protocol = url.split("/")[0] %}
{% set https_protocol = https_url.split("/")[0] %}
{% set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) %}
{% set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) %}
# TODO-Caddy upstream {{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ upstream }};
# TODO-Caddy
# TODO-Caddy pstream https_{{ slave_parameter.get('slave_reference') }} {
# TODO-Caddy server {{ https_upstream }};
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_http_port'] }};
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy
# TODO-Caddy server {
# TODO-Caddy listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_https_port'] }} ssl;
# TODO-Caddy
# TODO-Caddy server_name {{ slave_parameter.get('custom_domain') }};
# TODO-Caddy
# TODO-Caddy error_log {{ slave_parameter.get('error_log') }} error;
# TODO-Caddy access_log {{ slave_parameter.get('access_log') }} custom;
# TODO-Caddy
# TODO-Caddy ssl on;
# TODO-Caddy
# TODO-Caddy ssl_session_timeout 5m;
# TODO-Caddy ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# TODO-Caddy ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
# TODO-Caddy ssl_prefer_server_ciphers on;
# TODO-Caddy ssl_session_cache shared:SSL:10m;
# TODO-Caddy location /pub {
# TODO-Caddy push_stream_publisher;
# TODO-Caddy push_stream_channels_path $arg_id;
# TODO-Caddy # store messages in memory
# TODO-Caddy push_stream_store_messages off;
# TODO-Caddy
# TODO-Caddy # Message size limit
# TODO-Caddy # client_max_body_size MUST be equal to client_body_buffer_size or
# TODO-Caddy # you will be sorry.
# TODO-Caddy client_max_body_size 16k;
# TODO-Caddy client_body_buffer_size 16k;
# TODO-Caddy
# TODO-Caddy }
# TODO-Caddy
# TODO-Caddy location ~ /sub/(.*) {
# TODO-Caddy # activate subscriber mode for this location
# TODO-Caddy add_header "Access-Control-Allow-Origin" "*";
# TODO-Caddy add_header 'Access-Control-Allow-Credentials' 'false';
# TODO-Caddy add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
# TODO-Caddy add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
# TODO-Caddy
# TODO-Caddy push_stream_subscriber eventsource;
# TODO-Caddy # positional channel path
# TODO-Caddy push_stream_channels_path $1;
# TODO-Caddy
# TODO-Caddy # content-type
# TODO-Caddy default_type "text/event-stream; charset=utf-8";
# TODO-Caddy }
# TODO-Caddy}
{% endif %}
notfound.html 0000664 0000000 0000000 00000000327 13712426147 0032557 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates
Instance not found
This instance has not been found.
If this error persists, please check your instance URL and status on SlapOS Master.
replicate-publish-slave-information.cfg.in 0000664 0000000 0000000 00000007231 13712426147 0040173 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates {% set part_list = [] %}
{% set slave_information_dict = {} %}
# regroup slave information from all frontends
{% for frontend, slave_list_raw in slave_information.iteritems() %}
{% if slave_list_raw %}
{% set slave_list = json_module.loads(slave_list_raw) %}
{% else %}
{% set slave_list = [] %}
{% endif %}
{% for slave_dict in slave_list %}
{% set slave_reference = slave_dict.pop('slave-reference') %}
{% set log_access_url = slave_dict.pop('log-access', '') %}
{% set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
{% do current_slave_dict.update(slave_dict) %}
{% set log_access_list = current_slave_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend + ': ' + log_access_url) %}
{% do current_slave_dict.__setitem__(
'log-access-urls',
log_access_list
) %}
{% do current_slave_dict.__setitem__(
'replication_number',
current_slave_dict.get('replication_number', 0) + 1
) %}
{% do slave_information_dict.__setitem__(slave_reference, current_slave_dict) %}
{% endfor %}
{% endfor %}
{% for slave_reference, rejected_info_list in rejected_slave_information.iteritems() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('request-error-list', rejected_info_list) %}
{% endfor %}
{% for slave_reference, warning_info_list in warning_slave_information.iteritems() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('warning-list', warning_info_list) %}
{% endfor %}
{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).iteritems() %}
{% if slave_reference not in rejected_slave_information %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
{% endif %}
{% endfor %}
# Publish information for each slave
{% set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %}
{% for slave_reference, slave_information in slave_information_dict.iteritems() %}
{# Filter out destroyed, so not existing anymore, slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
{% if slave_reference in active_slave_instance_list %}
{% set publish_section_title = 'publish-%s' % slave_reference %}
{% do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
{% set log_access_url = slave_information.pop('log-access-urls', None) %}
{% if log_access_url %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{% endif %}
{% for key, value in slave_information.iteritems() %}
{{ key }} = {{ dumps(value) }}
{% endfor %}
{% endif %}
{% endfor %}
[buildout]
extends = {{ common_profile }}
parts =
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
rotate-script.sh.in 0000664 0000000 0000000 00000000702 13712426147 0033573 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!/bin/sh
set -e
LOG_DIR={{ log_dir }}
LOGROTATE_DIR={{ rotate_dir }}
PATTERN={{ pattern }}
COMPRESS={{ xz_binary }}
KEEP_DAYS={{ keep_days }}
# Move out ${PATTERN} files
find ${LOG_DIR} -maxdepth 1 -type f -name ${PATTERN} -exec mv {} ${LOGROTATE_DIR}/ \;
# Compress
find ${LOGROTATE_DIR} -maxdepth 1 -type f -name ${PATTERN} -exec ${COMPRESS} -9 {} \;
# Retent old files
find ${LOGROTATE_DIR} -maxdepth 1 -type f -mtime +${KEEP_DAYS} -delete
template-log-access.conf.in 0000664 0000000 0000000 00000001546 13712426147 0035146 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates # Access log configuration
{% for slave, directory in slave_log_directory.iteritems() %}
https://[{{ parameter_dict['global_ipv6'] }}]:{{ parameter_dict['https_port'] }}/{{ slave }}, https://{{ parameter_dict['local_ipv4'] }}:{{ parameter_dict['https_port'] }}/{{ slave }} {
bind {{ parameter_dict['local_ipv4'] }}
root {{ directory }}/
browse
tls {{ parameter_dict['ip_access_certificate'] }} {{ parameter_dict['ip_access_certificate'] }}
basicauth "{{ slave }}" {{ slave_password[slave] | trim }} {
"Log Access {{ slave }}"
/
}
log / {{ parameter_dict['access_log'] }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ parameter_dict['error_log'] }} {
rotate_size 0
* {{ parameter_dict['not_found_file'] }}
}
}
{% endfor %}
trafficserver/ 0000775 0000000 0000000 00000000000 13712426147 0032700 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates logging.config.jinja2 0000664 0000000 0000000 00000000413 13712426147 0036667 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates/trafficserver squid = format {
Format = '% % % %/% % % % % % %/% %'
}
log.ascii {
Format = squid,
Filename = 'squid',
RollingEnabled = 1,
RollingIntervalSec = 86400,
RollingOffsetHr = 0
}
records.config.jinja2 0000664 0000000 0000000 00000031567 13712426147 0036720 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates/trafficserver ##############################################################################
# *NOTE*: All options covered in this file should be documented in the docs:
#
# https://docs.trafficserver.apache.org/records.config
##############################################################################
##############################################################################
# SlapOS Specific configuration
##############################################################################
CONFIG proxy.config.proxy_name STRING {{ ats_configuration['hostname'] }}
CONFIG proxy.config.local_state_dir STRING {{ ats_directory['local-state'] }}
CONFIG proxy.config.config_dir STRING {{ ats_directory['configuration'] }}
CONFIG proxy.config.bin_path STRING {{ ats_directory['bin_path'] }}
CONFIG proxy.config.env_prep STRING example_prep.sh
CONFIG proxy.config.alarm_email STRING nobody
CONFIG proxy.config.syslog_facility STRING LOG_DAEMON
CONFIG proxy.config.output.logfile STRING traffic.out
CONFIG proxy.config.snapshot_dir STRING snapshots
CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
CONFIG proxy.config.admin.number_config_bak INT 0
CONFIG proxy.config.process_manager.mgmt_port INT {{ ats_configuration['mgmt-port'] }}
CONFIG proxy.config.admin.synthetic_port INT {{ ats_configuration['synthetic-port'] }}
LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
# Implement RFC 5861 with core
CONFIG proxy.config.http.cache.open_write_fail_action INT 2
CONFIG proxy.config.body_factory.template_sets_dir STRING {{ ats_configuration['templates-dir'] }}
# Support stale-if-error by returning cached content on backend 5xx or unavailability
CONFIG proxy.config.http.negative_revalidating_enabled INT 1
##############################################################################
# Proxy users variables. Docs:
# https://docs.trafficserver.apache.org/records.config#proxy-user-variables
##############################################################################
# Do not modify headers, as it needlessly pollutes information
CONFIG proxy.config.http.insert_client_ip INT 0
CONFIG proxy.config.http.insert_squid_x_forwarded_for INT 0
##############################################################################
# Thread configurations. Docs:
# https://docs.trafficserver.apache.org/records.config#thread-variables
##############################################################################
CONFIG proxy.config.exec_thread.autoconfig INT 1
CONFIG proxy.config.exec_thread.autoconfig.scale FLOAT 1.5
CONFIG proxy.config.exec_thread.limit INT 2
CONFIG proxy.config.accept_threads INT 1
CONFIG proxy.config.task_threads INT 2
CONFIG proxy.config.cache.threads_per_disk INT 8
CONFIG proxy.config.exec_thread.affinity INT 1
##############################################################################
# Specify server addresses and ports to bind for HTTP and HTTPS. Docs:
# https://docs.trafficserver.apache.org/records.config#proxy.config.http.server_ports
##############################################################################
CONFIG proxy.config.http.server_ports STRING {{ ats_configuration['local-ip'] + ':' + ats_configuration['input-port'] }}
##############################################################################
# Via: headers. Docs:
# https://docs.trafficserver.apache.org/records.config#proxy-config-http-insert-response-via-str
##############################################################################
CONFIG proxy.config.http.insert_request_via_str INT 1
CONFIG proxy.config.http.insert_response_via_str INT 0
##############################################################################
# Parent proxy configuration, in addition to these settings also see parent.config. Docs:
# https://docs.trafficserver.apache.org/records.config#parent-proxy-configuration
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/parent.config.en.html
##############################################################################
CONFIG proxy.config.http.parent_proxy_routing_enable INT 0
CONFIG proxy.config.http.parent_proxy.retry_time INT 300
CONFIG proxy.config.http.parent_proxy.connect_attempts_timeout INT 30
CONFIG proxy.config.http.forward.proxy_auth_to_parent INT 0
CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
##############################################################################
# HTTP connection timeouts (secs). Docs:
# https://docs.trafficserver.apache.org/records.config#http-connection-timeouts
##############################################################################
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120
CONFIG proxy.config.net.default_inactivity_timeout INT 86400
##############################################################################
# Origin server connect attempts. Docs:
# https://docs.trafficserver.apache.org/records.config#origin-server-connect-attempts
##############################################################################
# Try only once to connect (do not retry)
CONFIG proxy.config.http.connect_attempts_max_retries INT 0
# Try only once with server marked dead (do not retry)
CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 0
CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
CONFIG proxy.config.http.connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.post_connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.down_server.cache_time INT 60
CONFIG proxy.config.http.down_server.abort_threshold INT 10
##############################################################################
# Negative response caching, for redirects and errors. Docs:
# https://docs.trafficserver.apache.org/records.config#negative-response-caching
##############################################################################
CONFIG proxy.config.http.negative_caching_enabled INT 0
CONFIG proxy.config.http.negative_caching_lifetime INT 1800
##############################################################################
# Security. Docs:
# https://docs.trafficserver.apache.org/records.config#security
##############################################################################
CONFIG proxy.config.http.push_method_enabled INT 0
##############################################################################
# Enable / disable HTTP caching. Useful for testing, but also as an
# overridable (per remap) config
##############################################################################
CONFIG proxy.config.http.cache.http INT 1
##############################################################################
# Cache control. Docs:
# https://docs.trafficserver.apache.org/records.config#cache-control
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/cache.config.en.html
##############################################################################
CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 1
CONFIG proxy.config.http.normalize_ae_gzip INT 0
CONFIG proxy.config.http.cache.cache_responses_to_cookies INT 1
CONFIG proxy.config.http.cache.cache_urls_that_look_dynamic INT 1
# https://docs.trafficserver.apache.org/records.config#proxy-config-http-cache-when-to-revalidate
CONFIG proxy.config.http.cache.when_to_revalidate INT 0
# https://docs.trafficserver.apache.org/records.config#proxy-config-http-cache-required-headers
CONFIG proxy.config.http.cache.required_headers INT 2
##############################################################################
# Heuristic cache expiration. Docs:
# https://docs.trafficserver.apache.org/records.config#heuristic-expiration
##############################################################################
CONFIG proxy.config.http.cache.heuristic_min_lifetime INT 3600
CONFIG proxy.config.http.cache.heuristic_max_lifetime INT 86400
CONFIG proxy.config.http.cache.heuristic_lm_factor FLOAT 0.10
##############################################################################
# Network. Docs:
# https://docs.trafficserver.apache.org/records.config#network
##############################################################################
CONFIG proxy.config.net.connections_throttle INT 30000
CONFIG proxy.config.net.max_connections_in INT 30000
CONFIG proxy.config.net.max_connections_active_in INT 10000
##############################################################################
# RAM and disk cache configurations. Docs:
# https://docs.trafficserver.apache.org/records.config#ram-cache
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/storage.config.en.html
##############################################################################
CONFIG proxy.config.cache.ram_cache.size INT {{ ats_configuration.get('ram-cache-size', '1G') }}
CONFIG proxy.config.cache.ram_cache_cutoff INT 4194304
# https://docs.trafficserver.apache.org/records.config#proxy-config-cache-limits-http-max-alts
CONFIG proxy.config.cache.limits.http.max_alts INT 5
# https://docs.trafficserver.apache.org/records.config#proxy-config-cache-max-doc-size
CONFIG proxy.config.cache.max_doc_size INT 0
CONFIG proxy.config.cache.min_average_object_size INT 8000
##############################################################################
# Logging Config. Docs:
# https://docs.trafficserver.apache.org/records.config#logging-configuration
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.config.en.html
##############################################################################
CONFIG proxy.config.log.logging_enabled INT 3
CONFIG proxy.config.log.max_space_mb_for_logs INT 25000
CONFIG proxy.config.log.max_space_mb_headroom INT 1000
CONFIG proxy.config.log.rolling_enabled INT 1
CONFIG proxy.config.log.rolling_interval_sec INT 86400
CONFIG proxy.config.log.rolling_size_mb INT 10
CONFIG proxy.config.log.auto_delete_rolled_files INT 1
CONFIG proxy.config.log.periodic_tasks_interval INT 5
##############################################################################
# These settings control remapping, and if the proxy allows (open) forward proxy or not. Docs:
# https://docs.trafficserver.apache.org/records.config#url-remap-rules
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/remap.config.en.html
##############################################################################
CONFIG proxy.config.url_remap.remap_required INT 1
# https://docs.trafficserver.apache.org/records.config#proxy-config-url-remap-pristine-host-hdr
CONFIG proxy.config.url_remap.pristine_host_hdr INT 1
# https://docs.trafficserver.apache.org/records.config#reverse-proxy
CONFIG proxy.config.reverse_proxy.enabled INT 1
##############################################################################
# SSL Termination. Docs:
# https://docs.trafficserver.apache.org/records.config#client-related-configuration
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/ssl_multicert.config.en.html
##############################################################################
CONFIG proxy.config.ssl.client.verify.server INT 0
CONFIG proxy.config.ssl.client.CA.cert.filename STRING NULL
CONFIG proxy.config.ssl.server.cipher_suite STRING ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
##############################################################################
# Debugging. Docs:
# https://docs.trafficserver.apache.org/records.config#diagnostic-logging-configuration
##############################################################################
CONFIG proxy.config.diags.debug.enabled INT 0
CONFIG proxy.config.diags.debug.tags STRING http.*|dns.*
# ToDo: Undocumented
CONFIG proxy.config.dump_mem_info_frequency INT 0
CONFIG proxy.config.http.slow.log.threshold INT 0
storage.config.jinja2 0000664 0000000 0000000 00000000715 13712426147 0036712 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates/trafficserver #
# storage.config - Storage Configuration file
#
# Documentation:
# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/storage.config.en.html
#
# The storage configuration is a list of all the storage to
# be used by the cache.
#
#
#############################################################
# Using a file for the cache storage
#
#
{{ ats_configuration.get("cache-path") }} {{ ats_configuration.get("disk-cache-size") }}
validate-script.sh.in 0000664 0000000 0000000 00000000633 13712426147 0034071 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!/bin/sh
set -e
LAST_STATE_FILE={{ last_state_file }}
# force validation each 2 hours
old_found=1
if [ -f $LAST_STATE_FILE ] ; then
old_found=$(find $LAST_STATE_FILE -mmin +120 | wc -l)
fi
if [ "$old_found" -eq 1 ] || {{ configuration_state_command }} ; then
# do not catch errors during validation
set +e
{{ validate_command }}
echo $? > $LAST_STATE_FILE
set -e
fi
exit `cat $LAST_STATE_FILE`
wrapper.in 0000664 0000000 0000000 00000000044 13712426147 0032041 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/templates #!${dash-output:dash}
{{ content }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/ 0000775 0000000 0000000 00000000000 13712426147 0027073 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/setup.py 0000664 0000000 0000000 00000004373 13712426147 0030614 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.caddy-frontend'
setup(name=name,
version=version,
description="Test for SlapOS' Caddy Frontend",
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'erp5.util',
'requests >= 2.20.0', # needed for recent SSL certificate fixes
'urllib3 >= 1.24', # needed for recent SSL certificate fixes
# ipaddress is patching IPAddress so IPv6 in SSL certificates
# match works
'ipaddress >= 1.0.22',
'requests-toolbelt',
'supervisor',
# caucase needed to connect to the KeDiFa caucase
'caucase',
'cryptography',
'backports.lzma',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test.py 0000664 0000000 0000000 00000650072 13712426147 0030436 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import glob
import os
import requests
import httplib
from requests_toolbelt.adapters import source
import json
import multiprocessing
import subprocess
from unittest import skip
import ssl
from BaseHTTPServer import HTTPServer
from BaseHTTPServer import BaseHTTPRequestHandler
import time
import tempfile
import ipaddress
import StringIO
import gzip
import base64
import re
from slapos.recipe.librecipe import generateHashFromFiles
import xml.etree.ElementTree as ET
import urlparse
import socket
try:
import lzma
except ImportError:
from backports import lzma
import datetime
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# ports chosen to not collide with test systems
HTTP_PORT = '11080'
HTTPS_PORT = '11443'
CAUCASE_PORT = '15090'
KEDIFA_PORT = '15080'
# IP to originate requests from
# has to be not partition one
SOURCE_IP = '127.0.0.1'
# "--resolve" inspired from https://stackoverflow.com/a/44378047/9256748
DNS_CACHE = {}
def add_custom_dns(domain, port, ip):
port = int(port)
key = (domain, port)
value = (socket.AF_INET, 1, 6, '', (ip, port))
DNS_CACHE[key] = [value]
def new_getaddrinfo(*args):
return DNS_CACHE[args[:2]]
# for development: debugging logs and install Ctrl+C handler
if os.environ.get('SLAPOS_TEST_DEBUG'):
import logging
logging.basicConfig(level=logging.DEBUG)
import unittest
unittest.installHandler()
def der2pem(der):
certificate = x509.load_der_x509_certificate(der, default_backend())
return certificate.public_bytes(serialization.Encoding.PEM)
def createKey():
key = rsa.generate_private_key(
public_exponent=65537, key_size=2048, backend=default_backend())
key_pem = key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()
)
return key, key_pem
def createSelfSignedCertificate(name_list):
key, key_pem = createKey()
subject_alternative_name_list = x509.SubjectAlternativeName(
[x509.DNSName(unicode(q)) for q in name_list]
)
subject = issuer = x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'Test Self Signed Certificate'),
])
certificate = x509.CertificateBuilder().subject_name(
subject
).issuer_name(
issuer
).add_extension(
subject_alternative_name_list,
critical=False,
).public_key(
key.public_key()
).serial_number(
x509.random_serial_number()
).not_valid_before(
datetime.datetime.utcnow() - datetime.timedelta(days=2)
).not_valid_after(
datetime.datetime.utcnow() + datetime.timedelta(days=5)
).sign(key, hashes.SHA256(), default_backend())
certificate_pem = certificate.public_bytes(serialization.Encoding.PEM)
return key, key_pem, certificate, certificate_pem
def createCSR(common_name, ip=None):
key, key_pem = createKey()
subject_alternative_name_list = []
if ip is not None:
subject_alternative_name_list.append(
x509.IPAddress(ipaddress.ip_address(unicode(ip)))
)
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)),
]))
if len(subject_alternative_name_list):
csr = csr.add_extension(
x509.SubjectAlternativeName(subject_alternative_name_list),
critical=False
)
csr = csr.sign(key, hashes.SHA256(), default_backend())
csr_pem = csr.public_bytes(serialization.Encoding.PEM)
return key, key_pem, csr, csr_pem
class CertificateAuthority(object):
def __init__(self, common_name):
self.key, self.key_pem = createKey()
public_key = self.key.public_key()
builder = x509.CertificateBuilder()
builder = builder.subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)),
]))
builder = builder.issuer_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, unicode(common_name)),
]))
builder = builder.not_valid_before(
datetime.datetime.utcnow() - datetime.timedelta(days=2))
builder = builder.not_valid_after(
datetime.datetime.utcnow() + datetime.timedelta(days=30))
builder = builder.serial_number(x509.random_serial_number())
builder = builder.public_key(public_key)
builder = builder.add_extension(
x509.BasicConstraints(ca=True, path_length=None), critical=True,
)
self.certificate = builder.sign(
private_key=self.key, algorithm=hashes.SHA256(),
backend=default_backend()
)
self.certificate_pem = self.certificate.public_bytes(
serialization.Encoding.PEM)
def signCSR(self, csr):
builder = x509.CertificateBuilder(
subject_name=csr.subject,
extensions=csr.extensions,
issuer_name=self.certificate.subject,
not_valid_before=datetime.datetime.utcnow() - datetime.timedelta(days=1),
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=30),
serial_number=x509.random_serial_number(),
public_key=csr.public_key(),
)
certificate = builder.sign(
private_key=self.key,
algorithm=hashes.SHA256(),
backend=default_backend()
)
return certificate, certificate.public_bytes(serialization.Encoding.PEM)
def subprocess_status_output(*args, **kwargs):
prc = subprocess.Popen(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
*args,
**kwargs)
out, err = prc.communicate()
return prc.returncode, out
def subprocess_output(*args, **kwargs):
return subprocess_status_output(*args, **kwargs)[1]
def isHTTP2(domain, ip):
curl_command = 'curl --http2 -v -k -H "Host: %(domain)s" ' \
'https://%(domain)s:%(https_port)s/ '\
'--resolve %(domain)s:%(https_port)s:%(ip)s' % dict(
ip=ip, domain=domain, https_port=HTTPS_PORT)
prc = subprocess.Popen(
curl_command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
out, err = prc.communicate()
assert prc.returncode == 0, "Problem running %r. Output:\n%s\nError:\n%s" % (
curl_command, out, err)
return 'Using HTTP2, server supports multi-use' in err
def getPluginParameterDict(software_path, filepath):
"""Load the slapos monitor plugin and returns the configuration used by this plugin.
This allow to check that monitoring plugin are using a proper config.
"""
# This is implemented by creating a wrapper script that loads the plugin wrapper
# script and returns its `extra_config_dict`. This might have to be adjusted if
# internals of slapos promise plugins change.
bin_file = os.path.join(software_path, 'bin', 'test-plugin-promise')
monitor_python_with_eggs = os.path.join(software_path, 'bin', 'monitor-pythonwitheggs')
if not os.path.exists(monitor_python_with_eggs):
raise ValueError("Monitoring stack's python does not exist at %s" % monitor_python_with_eggs)
with open(bin_file, 'w') as f:
f.write("""#!%s
import os
import importlib
import sys
import json
filepath = sys.argv[1]
sys.path[0:0] = [os.path.dirname(filepath)]
filename = os.path.basename(filepath)
module = importlib.import_module(os.path.splitext(filename)[0])
print json.dumps(module.extra_config_dict)
""" % monitor_python_with_eggs)
os.chmod(bin_file, 0o755)
result = subprocess_output([bin_file, filepath]).strip()
try:
return json.loads(result)
except ValueError, e:
raise ValueError("%s\nResult was: %s" % (e, result))
class TestDataMixin(object):
def getTrimmedProcessInfo(self):
return '\n'.join(sorted([
'%(group)s:%(name)s %(statename)s' % q for q
in self.callSupervisorMethod('getAllProcessInfo')
if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
def assertTestData(self, runtime_data, hash_value_dict=None, msg=None):
if hash_value_dict is None:
hash_value_dict = {}
filename = '%s-%s.txt' % (self.id(), 'CADDY')
test_data_file = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'test_data', filename)
try:
test_data = open(test_data_file).read().strip()
except IOError:
test_data = ''
for hash_type, hash_value in hash_value_dict.items():
runtime_data = runtime_data.replace(hash_value, '{hash-%s}' % (
hash_type),)
maxDiff = self.maxDiff
self.maxDiff = None
longMessage = self.longMessage
self.longMessage = True
try:
self.assertMultiLineEqual(
test_data,
runtime_data,
msg=msg
)
except AssertionError:
if os.environ.get('SAVE_TEST_DATA', '0') == '1':
open(test_data_file, 'w').write(runtime_data.strip() + '\n')
raise
finally:
self.maxDiff = maxDiff
self.longMessage = longMessage
def _test_file_list(self, slave_dir_list, IGNORE_PATH_LIST=None):
if IGNORE_PATH_LIST is None:
IGNORE_PATH_LIST = []
runtime_data = []
for slave_var in glob.glob(os.path.join(self.instance_path, '*')):
for entry in os.walk(os.path.join(slave_var, *slave_dir_list)):
for filename in entry[2]:
path = os.path.join(
entry[0][len(self.instance_path) + 1:], filename)
if not any([path.endswith(q) for q in IGNORE_PATH_LIST]):
runtime_data.append(path)
runtime_data = '\n'.join(sorted(runtime_data))
self.assertTestData(runtime_data)
def test_file_list_log(self):
self._test_file_list(['var', 'log'], [
# no control at all when cron would kick in, ignore it
'cron.log',
# appears late and is quite unstable, no need to assert
'trafficserver/.diags.log.meta',
'trafficserver/.manager.log.meta',
'trafficserver/.squid.log.meta',
'trafficserver/.traffic.out.meta',
'trafficserver/diags.log',
'trafficserver/squid.log',
# not important, appears sometimes
'trafficserver/.error.log.meta',
'trafficserver/error.log',
])
def test_file_list_run(self):
self._test_file_list(['var', 'run'], [
# can't be sure regarding its presence
'caddy_configuration_last_state',
'validate_configuration_state_signature',
# run by cron from time to time
'monitor/monitor-collect.pid',
# no control regarding if it would or not be running
'monitor/monitor-bootstrap.pid',
])
def test_file_list_etc_cron_d(self):
self._test_file_list(['etc', 'cron.d'])
def test_file_list_plugin(self):
self._test_file_list(['etc', 'plugin'], ['.pyc'])
def test_supervisor_state(self):
# give a chance for etc/run scripts to finish
time.sleep(1)
hash_file_list = [os.path.join(
self.computer_partition_root_path, 'software_release/buildout.cfg')]
hash_value_dict = {
'generic': generateHashFromFiles(hash_file_list),
}
for caddy_wrapper_path in glob.glob(os.path.join(
self.instance_path, '*', 'bin', 'caddy-wrapper')):
partition_id = caddy_wrapper_path.split('/')[-3]
hash_value_dict[
'caddy-%s' % (partition_id)] = generateHashFromFiles(
[caddy_wrapper_path] + hash_file_list
)
for backend_haproxy_wrapper_path in glob.glob(os.path.join(
self.instance_path, '*', 'bin', 'backend-haproxy-wrapper')):
partition_id = backend_haproxy_wrapper_path.split('/')[-3]
hash_value_dict[
'backend-haproxy-%s' % (partition_id)] = generateHashFromFiles(
[backend_haproxy_wrapper_path] + hash_file_list
)
for rejected_slave_publish_path in glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave')):
partition_id = rejected_slave_publish_path.split('/')[-3]
rejected_slave_pem_path = os.path.join(
self.instance_path, partition_id, 'etc', 'rejected-slave.pem')
hash_value_dict[
'rejected-slave-publish'
] = generateHashFromFiles(
[rejected_slave_publish_path, rejected_slave_pem_path] + hash_file_list
)
runtime_data = self.getTrimmedProcessInfo()
self.assertTestData(runtime_data, hash_value_dict=hash_value_dict)
def fakeHTTPSResult(domain, real_ip, path, port=HTTPS_PORT,
headers=None, cookies=None, source_ip=SOURCE_IP):
if headers is None:
headers = {}
# workaround request problem of setting Accept-Encoding
# https://github.com/requests/requests/issues/2234
headers.setdefault('Accept-Encoding', 'dummy')
# Headers to tricks the whole system, like rouge user would do
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
session = requests.Session()
if source_ip is not None:
new_source = source.SourceAddressAdapter(source_ip)
session.mount('http://', new_source)
session.mount('https://', new_source)
socket_getaddrinfo = socket.getaddrinfo
try:
add_custom_dns(domain, port, real_ip)
socket.getaddrinfo = new_getaddrinfo
return session.get(
'https://%s:%s/%s' % (domain, port, path),
verify=False,
allow_redirects=False,
headers=headers,
cookies=cookies
)
finally:
socket.getaddrinfo = socket_getaddrinfo
def fakeHTTPResult(domain, real_ip, path, port=HTTP_PORT,
headers=None, source_ip=SOURCE_IP):
if headers is None:
headers = {}
# workaround request problem of setting Accept-Encoding
# https://github.com/requests/requests/issues/2234
headers.setdefault('Accept-Encoding', 'dummy')
# Headers to tricks the whole system, like rouge user would do
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
headers['Host'] = '%s:%s' % (domain, port)
session = requests.Session()
if source_ip is not None:
new_source = source.SourceAddressAdapter(source_ip)
session.mount('http://', new_source)
session.mount('https://', new_source)
return session.get(
'http://%s:%s/%s' % (real_ip, port, path),
headers=headers,
allow_redirects=False,
)
class TestHandler(BaseHTTPRequestHandler):
identification = None
def do_GET(self):
timeout = int(self.headers.dict.get('timeout', '0'))
compress = int(self.headers.dict.get('compress', '0'))
time.sleep(timeout)
self.send_response(200)
if self.identification is not None:
self.send_header('X-Backend-Identification', self.identification)
drop_header_list = []
for header in self.headers.dict.get('x-drop-header', '').split():
drop_header_list.append(header)
prefix = 'x-reply-header-'
length = len(prefix)
for key, value in self.headers.dict.items():
if key.startswith(prefix):
self.send_header(
'-'.join([q.capitalize() for q in key[length:].split('-')]),
value.strip()
)
if 'Content-Type' not in drop_header_list:
self.send_header("Content-Type", "application/json")
if 'Set-Cookie' not in drop_header_list:
self.send_header('Set-Cookie', 'secured=value;secure')
self.send_header('Set-Cookie', 'nonsecured=value')
if 'x-reply-body' not in self.headers.dict:
response = {
'Path': self.path,
'Incoming Headers': self.headers.dict
}
response = json.dumps(response, indent=2)
else:
response = base64.b64decode(self.headers.dict['x-reply-body'])
if compress:
self.send_header('Content-Encoding', 'gzip')
out = StringIO.StringIO()
# compress with level 0, to find out if in the middle someting would
# like to alter the compression
with gzip.GzipFile(fileobj=out, mode="w", compresslevel=0) as f:
f.write(response)
response = out.getvalue()
self.send_header('Backend-Content-Length', len(response))
if 'Content-Length' not in drop_header_list:
self.send_header('Content-Length', len(response))
self.end_headers()
self.wfile.write(response)
class HttpFrontendTestCase(SlapOSInstanceTestCase):
# show full diffs, as it is required for proper analysis of problems
maxDiff = None
# minimise partition path
__partition_reference__ = 'T-'
@classmethod
def getInstanceSoftwareType(cls):
# Because of unknown problem yet, the root instance software type changes
# from RootSoftwareInstance to '', so always request it with given type
return "RootSoftwareInstance"
@classmethod
def prepareCertificate(cls):
cls.another_server_ca = CertificateAuthority("Another Server Root CA")
cls.test_server_ca = CertificateAuthority("Test Server Root CA")
key, key_pem, csr, csr_pem = createCSR(
"testserver.example.com", cls._ipv4_address)
_, cls.test_server_certificate_pem = cls.test_server_ca.signCSR(csr)
cls.test_server_certificate_file = tempfile.NamedTemporaryFile(
delete=False
)
cls.test_server_certificate_file.write(
cls.test_server_certificate_pem + key_pem
)
cls.test_server_certificate_file.close()
@classmethod
def startServerProcess(cls):
server = HTTPServer(
(cls._ipv4_address, cls._server_http_port),
TestHandler)
server_https = HTTPServer(
(cls._ipv4_address, cls._server_https_port),
TestHandler)
server_https.socket = ssl.wrap_socket(
server_https.socket,
certfile=cls.test_server_certificate_file.name,
server_side=True)
cls.backend_url = 'http://%s:%s/' % server.server_address
cls.server_process = multiprocessing.Process(
target=server.serve_forever, name='HTTPServer')
cls.server_process.start()
cls.logger.debug('Started process %s' % (cls.server_process,))
cls.backend_https_url = 'https://%s:%s/' % server_https.server_address
cls.server_https_process = multiprocessing.Process(
target=server_https.serve_forever, name='HTTPSServer')
cls.server_https_process.start()
cls.logger.debug('Started process %s' % (cls.server_https_process,))
@classmethod
def cleanUpCertificate(cls):
if getattr(cls, 'test_server_certificate_file', None) is not None:
os.unlink(cls.test_server_certificate_file.name)
@classmethod
def stopServerProcess(cls):
for server in ['server_process', 'server_https_process']:
process = getattr(cls, server, None)
if process is not None:
cls.logger.debug('Stopping process %s' % (process,))
process.join(10)
process.terminate()
time.sleep(0.1)
if process.is_alive():
cls.logger.warning(
'Process %s still alive' % (process, ))
@classmethod
def setUpMaster(cls):
# run partition until AIKC finishes
cls.runComputerPartitionUntil(
cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
ca_certificate = requests.get(
parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
auth = requests.get(
parameter_dict['master-key-generate-auth-url'],
verify=cls.ca_certificate_file)
assert auth.status_code == httplib.CREATED
upload = requests.put(
parameter_dict['master-key-upload-url'] + auth.text,
data=cls.key_pem + cls.certificate_pem,
verify=cls.ca_certificate_file)
assert upload.status_code == httplib.CREATED
@classmethod
def runKedifaUpdater(cls):
kedifa_updater = None
for kedifa_updater in sorted(glob.glob(
os.path.join(
cls.instance_path, '*', 'etc', 'service', 'kedifa-updater*'))):
# fetch first kedifa-updater, as by default most of the tests are using
# only one running partition; in case if test does not need
# kedifa-updater this method can be overridden
break
if kedifa_updater is not None:
# try few times kedifa_updater
for i in range(10):
return_code, output = subprocess_status_output(
[kedifa_updater, '--once'])
if return_code == 0:
break
# wait for the other updater to work
time.sleep(2)
# assert that in the worst case last run was correct
assert return_code == 0, output
# give caddy a moment to refresh its config, as sending signal does not
# block until caddy is refreshed
time.sleep(2)
@classmethod
def createWildcardExampleComCertificate(cls):
_, cls.key_pem, _, cls.certificate_pem = createSelfSignedCertificate(
[
'*.customdomain.example.com',
'*.example.com',
'*.alias1.example.com',
])
@classmethod
def runComputerPartitionUntil(cls, until):
max_try = 10
try_num = 1
while True:
if until():
break
if try_num > max_try:
raise ValueError('Failed to run computer partition with %r' % (until,))
try:
cls.slap.waitForInstance()
except Exception:
cls.logger.exception("Error during until run")
try_num += 1
@classmethod
def untilNotReadyYetNotInMasterKeyGenerateAuthUrl(cls):
parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
key = 'master-key-generate-auth-url'
if key not in parameter_dict:
return False
if 'NotReadyYet' in parameter_dict[key]:
return False
return True
@classmethod
def callSupervisorMethod(cls, method, *args, **kwargs):
with cls.slap.instance_supervisor_rpc as instance_supervisor:
return getattr(instance_supervisor, method)(*args, **kwargs)
def assertRejectedSlavePromiseWithPop(self, parameter_dict):
rejected_slave_promise_url = parameter_dict.pop(
'rejected-slave-promise-url')
try:
result = requests.get(rejected_slave_promise_url, verify=False)
if result.text == '':
result_json = {}
else:
result_json = result.json()
self.assertEqual(
parameter_dict['rejected-slave-dict'],
result_json
)
except AssertionError:
raise
except Exception as e:
self.fail(e)
def assertLogAccessUrlWithPop(self, parameter_dict):
log_access_url = parameter_dict.pop('log-access-url')
self.assertTrue(len(log_access_url) >= 1)
# check only the first one, as second frontend will be stopped
log_access = log_access_url[0]
entry = log_access.split(': ')
if len(entry) != 2:
self.fail('Cannot parse %r' % (log_access,))
frontend, url = entry
result = requests.get(url, verify=False)
self.assertEqual(
httplib.OK,
result.status_code,
'While accessing %r of %r the status code was %r' % (
url, frontend, result.status_code))
self.assertEqual(
httplib.OK,
requests.get(url + 'access.log', verify=False).status_code
)
self.assertEqual(
httplib.OK,
requests.get(url + 'error.log', verify=False).status_code
)
# assert only for few tests, as backend log is not available for many of
# them, as it's created on the fly
for test_name in ['test_url', 'test_auth_to_backend', 'test_compressed_result']:
if self.id().endswith(test_name):
self.assertEqual(
httplib.OK,
requests.get(url + 'backend.log', verify=False).status_code
)
def assertKedifaKeysWithPop(self, parameter_dict, prefix=''):
generate_auth_url = parameter_dict.pop('%skey-generate-auth-url' % (
prefix,))
upload_url = parameter_dict.pop('%skey-upload-url' % (prefix,))
kedifa_ipv6_base = 'https://[%s]:%s' % (self._ipv6_address, KEDIFA_PORT)
base = '^' + kedifa_ipv6_base.replace(
'[', r'\[').replace(']', r'\]') + '/.{32}'
self.assertRegexpMatches(
generate_auth_url,
base + r'\/generateauth$'
)
self.assertRegexpMatches(
upload_url,
base + r'\?auth=$'
)
kedifa_caucase_url = parameter_dict.pop('kedifa-caucase-url')
self.assertEqual(
kedifa_caucase_url,
'http://[%s]:%s' % (self._ipv6_address, CAUCASE_PORT),
)
return generate_auth_url, upload_url
def assertKeyWithPop(self, key, d):
self.assertTrue(key in d, 'Key %r is missing in %r' % (key, d))
d.pop(key)
def assertEqualResultJson(self, result, key, value):
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertTrue(key in j, 'No key %r in %s' % (key, j))
self.assertEqual(value, j[key])
def patchRequests(self):
HTTPResponse = requests.packages.urllib3.response.HTTPResponse
HTTPResponse.orig__init__ = HTTPResponse.__init__
def new_HTTPResponse__init__(self, *args, **kwargs):
self.orig__init__(*args, **kwargs)
try:
self.peercert = self._connection.sock.getpeercert(binary_form=True)
except AttributeError:
pass
HTTPResponse.__init__ = new_HTTPResponse__init__
HTTPAdapter = requests.adapters.HTTPAdapter
HTTPAdapter.orig_build_response = HTTPAdapter.build_response
def new_HTTPAdapter_build_response(self, request, resp):
response = self.orig_build_response(request, resp)
try:
response.peercert = resp.peercert
except AttributeError:
pass
return response
HTTPAdapter.build_response = new_HTTPAdapter_build_response
def unpatchRequests(self):
HTTPResponse = requests.packages.urllib3.response.HTTPResponse
if getattr(HTTPResponse, 'orig__init__', None) is not None:
HTTPResponse.__init__ = HTTPResponse.orig__init__
del(HTTPResponse.orig__init__)
HTTPAdapter = requests.adapters.HTTPAdapter
if getattr(HTTPAdapter, 'orig_build_response', None) is not None:
HTTPAdapter.build_response = HTTPAdapter.orig_build_response
del(HTTPAdapter.orig_build_response)
def setUp(self):
# patch requests in order to being able to extract SSL certs
self.patchRequests()
def tearDown(self):
self.unpatchRequests()
super(HttpFrontendTestCase, self).tearDown()
def parseParameterDict(self, parameter_dict):
parsed_parameter_dict = {}
for key, value in parameter_dict.items():
if key in [
'rejected-slave-dict',
'warning-slave-dict',
'warning-list',
'request-error-list',
'log-access-url']:
value = json.loads(value)
parsed_parameter_dict[key] = value
return parsed_parameter_dict
def getMasterPartitionPath(self):
return '/' + os.path.join(
*glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave'
)
)[0].split('/')[:-2])
def parseConnectionParameterDict(self):
return self.parseParameterDict(
self.requestDefaultInstance().getConnectionParameterDict()
)
@classmethod
def waitForMethod(cls, name, method):
wait_time = 600
begin = time.time()
try_num = 0
cls.logger.debug('%s for %is' % (name, wait_time,))
while True:
try:
try_num += 1
method()
except Exception:
if time.time() - begin > wait_time:
cls.logger.exception(
"Error during %s after %.2fs" % (name, (time.time() - begin),))
raise
else:
time.sleep(0.5)
else:
cls.logger.info("%s took %.2fs" % (name, (time.time() - begin),))
break
@classmethod
def waitForCaddy(cls):
def method():
fakeHTTPSResult(
cls._ipv4_address, cls._ipv4_address,
'/',
)
cls.waitForMethod('waitForCaddy', method)
@classmethod
def _cleanup(cls, snapshot_name):
cls.cleanUpCertificate()
cls.stopServerProcess()
super(HttpFrontendTestCase, cls)._cleanup(snapshot_name)
@classmethod
def setUpClass(cls):
try:
cls.createWildcardExampleComCertificate()
cls.prepareCertificate()
# find ports once to be able startServerProcess many times
cls._server_http_port = findFreeTCPPort(cls._ipv4_address)
cls._server_https_port = findFreeTCPPort(cls._ipv4_address)
cls._server_https_auth_port = findFreeTCPPort(cls._ipv4_address)
cls.startServerProcess()
except BaseException:
cls.logger.exception("Error during setUpClass")
cls._cleanup("{}.{}.setUpClass".format(cls.__module__, cls.__name__))
cls.setUp = lambda self: self.fail('Setup Class failed.')
raise
super(HttpFrontendTestCase, cls).setUpClass()
try:
# expose instance directory
cls.instance_path = cls.slap.instance_directory
# expose software directory, extract from found computer partition
cls.software_path = os.path.realpath(os.path.join(
cls.computer_partition_root_path, 'software_release'))
# do working directory
cls.working_directory = os.path.join(os.path.realpath(
os.environ.get(
'SLAPOS_TEST_WORKING_DIR',
os.path.join(os.getcwd(), '.slapos'))),
'caddy-frontend-test')
if not os.path.isdir(cls.working_directory):
os.mkdir(cls.working_directory)
cls.setUpMaster()
cls.waitForCaddy()
except BaseException:
cls.logger.exception("Error during setUpClass")
# "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used
# by SlapOSInstanceTestCase.setUpClass so we use another name for
# snapshot, to make sure we don't store another snapshot in same
# directory.
cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(
cls.__module__, cls.__name__))
cls.setUp = lambda self: self.fail('Setup Class failed.')
raise
class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
@classmethod
def requestDefaultInstance(cls, state='started'):
default_instance = super(
SlaveHttpFrontendTestCase, cls).requestDefaultInstance(state=state)
if state != 'destroyed':
cls.requestSlaves()
return default_instance
@classmethod
def requestSlaves(cls):
software_url = cls.getSoftwareURL()
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
cls.logger.debug(
'requesting slave "%s" software:%s parameters:%s',
slave_reference, software_url, partition_parameter_kw)
cls.slap.request(
software_release=software_url,
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
@classmethod
def setUpClass(cls):
super(SlaveHttpFrontendTestCase, cls).setUpClass()
try:
cls.setUpSlaves()
cls.waitForSlave()
except BaseException:
cls.logger.exception("Error during setUpClass")
# "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used
# by SlapOSInstanceTestCase.setUpClass so we use another name for
# snapshot, to make sure we don't store another snapshot in same
# directory.
cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(
cls.__module__, cls.__name__))
cls.setUp = lambda self: self.fail('Setup Class failed.')
raise
@classmethod
def waitForSlave(cls):
def method():
for parameter_dict in cls.getSlaveConnectionParameterDictList():
if 'domain' in parameter_dict and 'public-ipv4' in parameter_dict:
try:
fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], '/')
except requests.exceptions.InvalidURL:
# ignore slaves to which connection is impossible by default
continue
cls.waitForMethod('waitForSlave', method)
@classmethod
def getSlaveConnectionParameterDictList(cls):
parameter_dict_list = []
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
parameter_dict_list.append(cls.slap.request(
software_release=cls.getSoftwareURL(),
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
).getConnectionParameterDict())
return parameter_dict_list
@classmethod
def untilSlavePartitionReady(cls):
# all on-watch services shall not be exited
for process in cls.callSupervisorMethod('getAllProcessInfo'):
if process['name'].endswith('-on-watch') and \
process['statename'] == 'EXITED':
if process['name'].startswith('monitor-http'):
continue
return False
for parameter_dict in cls.getSlaveConnectionParameterDictList():
log_access_ready = 'log-access-url' in parameter_dict
key = 'key-generate-auth-url'
key_generate_auth_ready = key in parameter_dict \
and 'NotReadyYet' not in parameter_dict[key]
if not(log_access_ready and key_generate_auth_ready):
return False
return True
@classmethod
def setUpSlaves(cls):
cls.runComputerPartitionUntil(
cls.untilSlavePartitionReady)
cls.updateSlaveConnectionParameterDictDict()
@classmethod
def updateSlaveConnectionParameterDictDict(cls):
cls.slave_connection_parameter_dict_dict = {}
# run partition for slaves to be setup
request = cls.slap.request
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
slave_instance = request(
software_release=cls.getSoftwareURL(),
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
cls.slave_connection_parameter_dict_dict[slave_reference] = \
slave_instance.getConnectionParameterDict()
def parseSlaveParameterDict(self, key):
return self.parseParameterDict(
self.slave_connection_parameter_dict_dict[
key
]
)
def assertSlaveBase(self, reference):
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = reference.translate(None, '_-').lower()
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
return parameter_dict
class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
def test(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertEqual(
{
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '0',
'rejected-slave-amount': '0',
'slave-amount': '0',
'rejected-slave-dict': {}
},
parameter_dict
)
class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
def test(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertEqual(
{
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'None',
'accepted-slave-amount': '0',
'rejected-slave-amount': '0',
'slave-amount': '0',
'rejected-slave-dict': {}},
parameter_dict
)
class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
'request-timeout': '12',
}
@classmethod
def prepareCertificate(cls):
cls.ca = CertificateAuthority('TestSlave')
_, cls.customdomain_ca_key_pem, csr, _ = createCSR(
'customdomainsslcrtsslkeysslcacrt.example.com')
_, cls.customdomain_ca_certificate_pem = cls.ca.signCSR(csr)
_, cls.customdomain_key_pem, _, cls.customdomain_certificate_pem = \
createSelfSignedCertificate(['customdomainsslcrtsslkey.example.com'])
super(TestSlave, cls).prepareCertificate()
@classmethod
def getSlaveParameterDictDict(cls):
return {
'empty': {
},
'Url': {
# make URL "incorrect", with whitespace, nevertheless it shall be
# correctly handled
'url': ' ' + cls.backend_url + ' ',
# authenticating to http backend shall be no-op
'authenticate-to-backend': True,
},
'auth-to-backend': {
# in here use reserved port for the backend, which is going to be
# started later
'url': 'https://%s:%s/' % (
cls._ipv4_address, cls._server_https_auth_port),
'authenticate-to-backend': True,
},
'auth-to-backend-not-configured': {
# in here use reserved port for the backend, which is going to be
# started later
'url': 'https://%s:%s/' % (
cls._ipv4_address, cls._server_https_auth_port),
},
'auth-to-backend-backend-ignore': {
'url': cls.backend_https_url,
'authenticate-to-backend': True,
},
'url_https-url': {
'url': cls.backend_url + 'http',
'https-url': cls.backend_url + 'https',
'backend-connect-timeout': 10,
'backend-connect-retries': 5,
'request-timeout': 15,
},
'server-alias': {
'url': cls.backend_url,
'server-alias': 'alias1.example.com alias2.example.com',
},
'server-alias-empty': {
'url': cls.backend_url,
'server-alias': '',
},
'server-alias-wildcard': {
'url': cls.backend_url,
'server-alias': '*.alias1.example.com',
},
'server-alias-duplicated': {
'url': cls.backend_url,
'server-alias': 'alias3.example.com alias3.example.com',
},
'server-alias_custom_domain-duplicated': {
'url': cls.backend_url,
'custom_domain': 'alias4.example.com',
'server-alias': 'alias4.example.com alias4.example.com',
},
'ssl-proxy-verify_ssl_proxy_ca_crt': {
'url': cls.backend_https_url,
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': cls.test_server_ca.certificate_pem,
},
'ssl-proxy-verify_ssl_proxy_ca_crt-unverified': {
'url': cls.backend_https_url,
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': cls.another_server_ca.certificate_pem,
},
'ssl-proxy-verify-unverified': {
'url': cls.backend_https_url,
'ssl-proxy-verify': True,
},
'https-only': {
'url': cls.backend_url,
'https-only': False,
},
'custom_domain': {
'url': cls.backend_url,
'custom_domain': 'mycustomdomain.example.com',
},
'custom_domain_wildcard': {
'url': cls.backend_url,
'custom_domain': '*.customdomain.example.com',
},
'custom_domain_server_alias': {
'url': cls.backend_url,
'custom_domain': 'mycustomdomainserveralias.example.com',
'server-alias': 'mycustomdomainserveralias1.example.com',
},
'custom_domain_ssl_crt_ssl_key': {
'url': cls.backend_url,
'custom_domain': 'customdomainsslcrtsslkey.example.com',
},
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt': {
'url': cls.backend_url,
'custom_domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
},
'ssl_ca_crt_only': {
'url': cls.backend_url,
},
'ssl_ca_crt_garbage': {
'url': cls.backend_url,
},
'ssl_ca_crt_does_not_match': {
'url': cls.backend_url,
},
'type-zope': {
'url': cls.backend_url,
'type': 'zope',
},
'type-zope-prefer-gzip-encoding-to-backend': {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
'type': 'zope',
},
'type-zope-prefer-gzip-encoding-to-backend-https-only': {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
'type': 'zope',
'https-only': 'false',
},
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt': {
'url': cls.backend_https_url,
'type': 'zope',
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': cls.test_server_ca.certificate_pem,
},
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified': {
'url': cls.backend_https_url,
'type': 'zope',
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': cls.another_server_ca.certificate_pem,
},
'type-zope-ssl-proxy-verify-unverified': {
'url': cls.backend_https_url,
'type': 'zope',
'ssl-proxy-verify': True,
},
'type-zope-virtualhostroot-http-port': {
'url': cls.backend_url,
'type': 'zope',
'virtualhostroot-http-port': '12345',
'https-only': 'false',
},
'type-zope-virtualhostroot-https-port': {
'url': cls.backend_url,
'type': 'zope',
'virtualhostroot-https-port': '12345'
},
'type-zope-path': {
'url': cls.backend_url,
'type': 'zope',
'path': '///path/to/some/resource///',
},
'type-zope-default-path': {
'url': cls.backend_url,
'type': 'zope',
'default-path': '///default-path/to/some/resource///',
},
'type-notebook': {
'url': cls.backend_url,
'type': 'notebook',
},
'type-websocket': {
'url': cls.backend_url,
'type': 'websocket',
},
'type-websocket-websocket-path-list': {
'url': cls.backend_url,
'type': 'websocket',
'websocket-path-list': '////ws//// /with%20space/',
},
'type-websocket-websocket-transparent-false': {
'url': cls.backend_url,
'type': 'websocket',
'websocket-transparent': 'false',
},
'type-websocket-websocket-path-list-websocket-transparent-false': {
'url': cls.backend_url,
'type': 'websocket',
'websocket-path-list': '////ws//// /with%20space/',
'websocket-transparent': 'false',
},
# 'type-eventsource': {
# 'url': cls.backend_url,
# 'type': 'eventsource',
# },
'type-redirect': {
'url': cls.backend_url,
'type': 'redirect',
},
'type-redirect-custom_domain': {
'url': cls.backend_url,
'type': 'redirect',
'custom_domain': 'customdomaintyperedirect.example.com',
},
'enable_cache': {
'url': cls.backend_url,
'enable_cache': True,
},
'enable_cache_custom_domain': {
'url': cls.backend_url,
'enable_cache': True,
'custom_domain': 'customdomainenablecache.example.com',
},
'enable_cache_server_alias': {
'url': cls.backend_url,
'enable_cache': True,
'server-alias': 'enablecacheserveralias1.example.com',
},
'enable_cache-disable-no-cache-request': {
'url': cls.backend_url,
'enable_cache': True,
'disable-no-cache-request': True,
},
'enable_cache-disable-via-header': {
'url': cls.backend_url,
'enable_cache': True,
'disable-via-header': True,
},
'enable-http2-false': {
'url': cls.backend_url,
'enable-http2': False,
},
'enable_cache-ssl-proxy-verify-unverified': {
'url': cls.backend_https_url,
'enable_cache': True,
'ssl-proxy-verify': True,
},
'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt': {
'url': cls.backend_https_url,
'enable_cache': True,
'ssl_proxy_ca_crt': cls.test_server_ca.certificate_pem,
'ssl-proxy-verify': True,
},
'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified': {
'url': cls.backend_https_url,
'enable_cache': True,
'ssl_proxy_ca_crt': cls.another_server_ca.certificate_pem,
'ssl-proxy-verify': True,
},
'enable-http2-default': {
'url': cls.backend_url,
},
'prefer-gzip-encoding-to-backend': {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
},
'prefer-gzip-encoding-to-backend-https-only': {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
'https-only': 'false',
},
'disabled-cookie-list': {
'url': cls.backend_url,
'disabled-cookie-list': 'Chocolate Vanilia',
},
'monitor-ipv4-test': {
'monitor-ipv4-test': 'monitor-ipv4-test',
},
'monitor-ipv6-test': {
'monitor-ipv6-test': 'monitor-ipv6-test',
},
'ciphers': {
'ciphers': 'RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA',
}
}
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
IP = self._ipv6_address
self.monitor_configuration_list = [
{
'htmlUrl': 'https://[%s]:8401/public/feed' % (IP,),
'text': 'testing partition 0',
'title': 'testing partition 0',
'type': 'rss',
'url': 'https://[%s]:8401/share/private/' % (IP,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:8401/public/feed' % (IP,),
},
{
'htmlUrl': 'https://[%s]:8402/public/feed' % (IP,),
'text': 'kedifa',
'title': 'kedifa',
'type': 'rss',
'url': 'https://[%s]:8402/share/private/' % (IP,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:8402/public/feed' % (IP,),
},
{
'htmlUrl': 'https://[%s]:8411/public/feed' % (IP,),
'text': 'caddy-frontend-1',
'title': 'caddy-frontend-1',
'type': 'rss',
'url': 'https://[%s]:8411/share/private/' % (IP,),
'version': 'RSS',
'xmlUrl': 'https://[%s]:8411/public/feed' % (IP,),
},
]
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
self.assertTrue(
self.monitor_setup_url_key in connection_parameter_dict,
'%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
monitor_setup_url_value = connection_parameter_dict[
self.monitor_setup_url_key]
monitor_url_match = re.match(r'.*url=(.*)', monitor_setup_url_value)
self.assertNotEqual(
None, monitor_url_match, '%s not parsable' % (monitor_setup_url_value,))
self.assertEqual(1, len(monitor_url_match.groups()))
monitor_url = monitor_url_match.groups()[0]
monitor_url_split = monitor_url.split('&')
self.assertEqual(
3, len(monitor_url_split), '%s not splitabble' % (monitor_url,))
self.monitor_url = monitor_url_split[0]
monitor_username = monitor_url_split[1].split('=')
self.assertEqual(
2, len(monitor_username), '%s not splittable' % (monitor_username))
monitor_password = monitor_url_split[2].split('=')
self.assertEqual(
2, len(monitor_password), '%s not splittable' % (monitor_password))
self.monitor_username = monitor_username[1]
self.monitor_password = monitor_password[1]
opml_text = requests.get(self.monitor_url, verify=False).text
opml = ET.fromstring(opml_text)
body = opml[1]
self.assertEqual('body', body.tag)
outline_list = body[0].findall('outline')
self.assertEqual(
self.monitor_configuration_list,
[q.attrib for q in outline_list]
)
expected_status_code_list = []
got_status_code_list = []
for monitor_configuration in self.monitor_configuration_list:
status_code = requests.get(
monitor_configuration['url'],
verify=False,
auth=(self.monitor_username, self.monitor_password)
).status_code
expected_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': 200
}
)
got_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': status_code
}
)
self.assertEqual(
expected_status_code_list,
got_status_code_list
)
def getMasterPartitionPath(self):
# partition w/o etc/trafficserver, but with buildout.cfg
return [
q for q in glob.glob(os.path.join(self.instance_path, '*',))
if not os.path.exists(
os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
os.path.join(q, 'buildout.cfg'))][0]
def getSlavePartitionPath(self):
# partition w/ etc/trafficserver
return [
q for q in glob.glob(os.path.join(self.instance_path, '*',))
if os.path.exists(os.path.join(q, 'etc', 'trafficserver'))][0]
def test_trafficserver_logrotate(self):
ats_partition = [
q for q in glob.glob(os.path.join(self.instance_path, '*',))
if os.path.exists(os.path.join(q, 'bin', 'trafficserver-rotate'))][0]
ats_log_dir = os.path.join(ats_partition, 'var', 'log', 'trafficserver')
ats_logrotate_dir = os.path.join(
ats_partition, 'srv', 'backup', 'logrotate', 'trafficserver')
ats_rotate = os.path.join(ats_partition, 'bin', 'trafficserver-rotate')
old_file_name = 'log-old.old'
older_file_name = 'log-older.old'
with open(os.path.join(ats_log_dir, old_file_name), 'w') as fh:
fh.write('old')
with open(os.path.join(ats_log_dir, older_file_name), 'w') as fh:
fh.write('older')
# check rotation
result, output = subprocess_status_output([ats_rotate])
self.assertEqual(0, result)
self.assertEqual(
set(['log-old.old.xz', 'log-older.old.xz']),
set(os.listdir(ats_logrotate_dir)))
self.assertFalse(old_file_name + '.xz' in os.listdir(ats_log_dir))
self.assertFalse(older_file_name + '.xz' in os.listdir(ats_log_dir))
with lzma.open(
os.path.join(ats_logrotate_dir, old_file_name + '.xz')) as fh:
self.assertEqual(
'old',
fh.read()
)
with lzma.open(
os.path.join(ats_logrotate_dir, older_file_name + '.xz')) as fh:
self.assertEqual(
'older',
fh.read()
)
# check retention
old_time = time.time() - (400 * 24 * 3600)
os.utime(
os.path.join(ats_logrotate_dir, older_file_name + '.xz'),
(old_time, old_time))
result, output = subprocess_status_output([ats_rotate])
self.assertEqual(0, result)
self.assertEqual(
['log-old.old.xz'],
os.listdir(ats_logrotate_dir))
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '56',
'rejected-slave-amount': '0',
'slave-amount': '56',
'rejected-slave-dict': {
},
'warning-slave-dict': {
'_Url': [
"slave url ' %(backend)s ' has been converted to '%(backend)s'" % {
'backend': self.backend_url}]}
}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
partition_path = self.getMasterPartitionPath()
# check that monitor cors domains are correctly setup by file presence, as
# we trust monitor stack being tested in proper place and it is too hard
# to have working monitor with local proxy
self.assertTestData(
open(
os.path.join(
partition_path, 'etc', 'httpd-cors.cfg'), 'r').read().strip())
def test_slave_partition_state(self):
partition_path = self.getSlavePartitionPath()
self.assertTrue(
'-grace 2s' in
open(os.path.join(partition_path, 'bin', 'caddy-wrapper'), 'r').read()
)
def test_monitor_conf(self):
monitor_conf_list = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'monitor.conf'
))
self.assertEqual(3, len(monitor_conf_list))
expected = [(False, q) for q in monitor_conf_list]
got = [('!py!' in open(q).read(), q) for q in monitor_conf_list]
# check that no monitor.conf in generated configuratio has magic !py!
self.assertEqual(
expected,
got
)
def test_empty(self):
parameter_dict = self.assertSlaveBase('empty')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://empty.example.com:%s/test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
# check that 404 is as configured
result_missing = fakeHTTPSResult(
'forsuredoesnotexists.example.com', parameter_dict['public-ipv4'], '')
self.assertEqual(httplib.NOT_FOUND, result_missing.status_code)
self.assertEqual(
"""
Instance not found
This instance has not been found.
If this error persists, please check your instance URL and status on S"""
"""lapOS Master.
""",
result_missing.text
)
def test_server_polluted_keys_removed(self):
buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
for line in [
q for q in open(buildout_file).readlines()
if q.startswith('config-slave-list') or q.startswith(
'config-extra_slave_instance_list')]:
self.assertFalse('slave_title' in line)
self.assertFalse('slap_software_type' in line)
self.assertFalse('connection-parameter-hash' in line)
self.assertFalse('timestamp' in line)
def assertBackendHeaders(
self, backend_header_dict, domain, source_ip=SOURCE_IP, port=HTTPS_PORT,
proto='https', ignore_header_list=None):
if ignore_header_list is None:
ignore_header_list = []
self.assertFalse('remote_user' in backend_header_dict.keys())
if 'Host' not in ignore_header_list:
self.assertEqual(
backend_header_dict['host'],
'%s:%s' % (domain, port))
self.assertEqual(
backend_header_dict['x-forwarded-for'],
source_ip
)
self.assertEqual(
backend_header_dict['x-forwarded-port'],
port
)
self.assertEqual(
backend_header_dict['x-forwarded-proto'],
proto
)
def test_url(self):
parameter_dict = self.assertSlaveBase('Url')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertEqual(j['Incoming Headers']['timeout'], '10')
self.assertFalse('Content-Encoding' in result.headers)
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
)
# check access log
log_file = glob.glob(
os.path.join(
self.instance_path, '*', 'var', 'log', 'httpd', '_Url_access_log'
))[0]
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
r'\d+ "-" "python-requests.*" \d+'
self.assertRegexpMatches(
open(log_file, 'r').readlines()[-1],
log_regexp)
# check backend log
log_file = glob.glob(
os.path.join(
self.instance_path, '*', 'var', 'log', 'httpd', '_Url_backend_log'
))[0]
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+ ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2}.\d{3}\] ' \
r'http-backend _Url-http\/backend ' \
r'\d+/\d+\/\d+\/\d+\/\d+ ' \
r'200 \d+ - - ---- ' \
r'\d\/\d\/\d\/\d\/\d \d\/\d ' \
r'"GET /test-path/deeper HTTP/1.1"'
self.assertRegexpMatches(
open(log_file, 'r').readlines()[-1],
log_regexp)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://url.example.com:%s/test-path/deeper' % (HTTP_PORT,),
result_http.headers['Location']
)
# check that timeouts are correctly set in the haproxy configuration
backend_configuration_file = glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
content = fh.read()
self.assertTrue("""backend _Url-http
timeout server 12s
timeout connect 5s
retries 3""" in content)
self.assertTrue(""" timeout queue 60s
timeout server 12s
timeout client 12s
timeout connect 5s
retries 3""" in content)
def test_auth_to_backend(self):
parameter_dict = self.assertSlaveBase('auth-to-backend')
# 1. fetch certificate from backend-client-caucase-url
master_parameter_dict = self.parseConnectionParameterDict()
caucase_url = master_parameter_dict['backend-client-caucase-url']
ca_certificate = requests.get(caucase_url + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
ca_certificate_file = os.path.join(
self.working_directory, 'ca-backend-client.crt.pem')
with open(ca_certificate_file, 'w') as fh:
fh.write(ca_certificate.text)
# 2. start backend with this certificate
class OwnTestHandler(TestHandler):
identification = 'Auth Backend'
server_https_auth = HTTPServer(
(self._ipv4_address, self._server_https_auth_port),
OwnTestHandler)
server_https_auth.socket = ssl.wrap_socket(
server_https_auth.socket,
certfile=self.test_server_certificate_file.name,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=ca_certificate_file,
server_side=True)
backend_https_auth_url = 'https://%s:%s/' \
% server_https_auth.server_address
server_https_auth_process = multiprocessing.Process(
target=server_https_auth.serve_forever, name='HTTPSServerAuth')
server_https_auth_process.start()
self.logger.debug('Started process %s' % (server_https_auth_process,))
try:
# 3. assert that you can't fetch nothing without key
try:
requests.get(backend_https_auth_url, verify=False)
except Exception:
pass
else:
self.fail(
'Access to %r shall be not possible without certificate' % (
backend_https_auth_url,))
# 4. check that you can access this backend via frontend
# (so it means that auth to backend worked)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertEqual(j['Incoming Headers']['timeout'], '10')
self.assertFalse('Content-Encoding' in result.headers)
self.assertBackendHeaders(
j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
)
# proof that proper backend was accessed
self.assertEqual(
'Auth Backend',
result.headers['X-Backend-Identification']
)
finally:
self.logger.debug('Stopping process %s' % (server_https_auth_process,))
server_https_auth_process.join(10)
server_https_auth_process.terminate()
time.sleep(0.1)
if server_https_auth_process.is_alive():
self.logger.warning(
'Process %s still alive' % (server_https_auth_process, ))
def test_auth_to_backend_not_configured(self):
parameter_dict = self.assertSlaveBase('auth-to-backend-not-configured')
# 1. fetch certificate from backend-client-caucase-url
master_parameter_dict = self.parseConnectionParameterDict()
caucase_url = master_parameter_dict['backend-client-caucase-url']
ca_certificate = requests.get(caucase_url + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
ca_certificate_file = os.path.join(
self.working_directory, 'ca-backend-client.crt.pem')
with open(ca_certificate_file, 'w') as fh:
fh.write(ca_certificate.text)
# 2. start backend with this certificate
class OwnTestHandler(TestHandler):
identification = 'Auth Backend'
server_https_auth = HTTPServer(
(self._ipv4_address, self._server_https_auth_port),
OwnTestHandler)
server_https_auth.socket = ssl.wrap_socket(
server_https_auth.socket,
certfile=self.test_server_certificate_file.name,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=ca_certificate_file,
server_side=True)
backend_https_auth_url = 'https://%s:%s/' \
% server_https_auth.server_address
server_https_auth_process = multiprocessing.Process(
target=server_https_auth.serve_forever, name='HTTPSServerAuth')
server_https_auth_process.start()
self.logger.debug('Started process %s' % (server_https_auth_process,))
try:
# 3. assert that you can't fetch nothing without key
try:
requests.get(backend_https_auth_url, verify=False)
except Exception:
pass
else:
self.fail(
'Access to %r shall be not possible without certificate' % (
backend_https_auth_url,))
# 4. check that you can access this backend via frontend
# (so it means that auth to backend worked)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
result.status_code,
httplib.BAD_GATEWAY
)
finally:
self.logger.debug('Stopping process %s' % (server_https_auth_process,))
server_https_auth_process.join(10)
server_https_auth_process.terminate()
time.sleep(0.1)
if server_https_auth_process.is_alive():
self.logger.warning(
'Process %s still alive' % (server_https_auth_process, ))
def test_auth_to_backend_backend_ignore(self):
parameter_dict = self.assertSlaveBase('auth-to-backend-backend-ignore')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertEqual(j['Incoming Headers']['timeout'], '10')
self.assertFalse('Content-Encoding' in result.headers)
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://authtobackendbackendignore.example.com:%s/test-path/deeper' % (
HTTP_PORT,),
result_http.headers['Location']
)
def test_compressed_result(self):
parameter_dict = self.assertSlaveBase('Url')
result_compressed = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Accept-Encoding': 'gzip',
'Compress': '1',
}
)
self.assertEqual(
'gzip',
result_compressed.headers['Content-Encoding']
)
# Assert that no tampering was done with the request
# (compression/decompression)
# Backend compresses with 0 level, so decompression/compression
# would change somthing
self.assertEqual(
result_compressed.headers['Content-Length'],
result_compressed.headers['Backend-Content-Length']
)
result_not_compressed = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Accept-Encoding': 'gzip',
}
)
self.assertFalse('Content-Encoding' in result_not_compressed.headers)
def test_no_content_type_alter(self):
parameter_dict = self.assertSlaveBase('Url')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Accept-Encoding': 'gzip',
'X-Reply-Body': base64.b64encode(
b"""
Tove
Jani
Reminder
Don't forget me this weekend!
"""),
'X-Drop-Header': 'Content-Type'
}
)
self.assertEqual(
'text/xml; charset=utf-8',
result.headers['Content-Type']
)
@skip('Feature postponed')
def test_url_ipv6_access(self):
parameter_dict = self.parseSlaveParameterDict('url')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'url.example.com',
'replication_number': '1',
'url': 'http://url.example.com',
'site_url': 'http://url.example.com',
'secure_access': 'https://url.example.com',
'public-ipv4': self._ipv4_address,
},
parameter_dict
)
result_ipv6 = fakeHTTPSResult(
parameter_dict['domain'], self._ipv6_address, 'test-path',
source_ip=self._ipv6_address)
self.assertEqual(
self._ipv6_address,
result_ipv6.json()['Incoming Headers']['x-forwarded-for']
)
self.assertEqual(
self.certificate_pem,
der2pem(result_ipv6.peercert))
self.assertEqualResultJson(result_ipv6, 'Path', '/test-path')
def test_type_zope_path(self):
parameter_dict = self.assertSlaveBase('type-zope-path')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/'
'https//typezopepath.example.com:443/path/to/some/resource'
'/VirtualHostRoot/'
'test-path/deeper'
)
def test_type_zope_default_path(self):
parameter_dict = self.assertSlaveBase('type-zope-default-path')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], '')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.MOVED_PERMANENTLY,
result.status_code
)
self.assertEqual(
'https://typezopedefaultpath.example.com:%s/'
'default-path/to/some/resource' % (
HTTPS_PORT,),
result.headers['Location']
)
def test_server_alias(self):
parameter_dict = self.assertSlaveBase('server-alias')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPSResult(
'alias1.example.com', parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPSResult(
'alias2.example.com', parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
def test_server_alias_empty(self):
parameter_dict = self.assertSlaveBase('server-alias-empty')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertEqual(j['Incoming Headers']['timeout'], '10')
self.assertFalse('Content-Encoding' in result.headers)
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
)
def test_server_alias_wildcard(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-wildcard')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliaswildcard.example.com',
'replication_number': '1',
'url': 'http://serveraliaswildcard.example.com',
'site_url': 'http://serveraliaswildcard.example.com',
'secure_access': 'https://serveraliaswildcard.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
result = fakeHTTPSResult(
'wild.alias1.example.com', parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_server_alias_duplicated(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-duplicated')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliasduplicated.example.com',
'replication_number': '1',
'url': 'http://serveraliasduplicated.example.com',
'site_url': 'http://serveraliasduplicated.example.com',
'secure_access': 'https://serveraliasduplicated.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
result = fakeHTTPSResult(
'alias3.example.com', parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_server_alias_custom_domain_duplicated(self):
parameter_dict = self.parseSlaveParameterDict(
'server-alias_custom_domain-duplicated')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'alias4.example.com',
'replication_number': '1',
'url': 'http://alias4.example.com',
'site_url': 'http://alias4.example.com',
'secure_access': 'https://alias4.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
@skip('Feature postponed')
def test_check_error_log(self):
# Caddy: Need to implement similar thing like check-error-on-apache-log
raise NotImplementedError(self.id())
def test_ssl_ca_crt(self):
parameter_dict = self.parseSlaveParameterDict(
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
'replication_number': '1',
'url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'site_url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'secure_access':
'https://customdomainsslcrtsslkeysslcacrt.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.customdomain_ca_certificate_pem + \
self.customdomain_ca_key_pem + \
self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.customdomain_ca_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'autocert',
'_custom_domain_ssl_crt_ssl_key_ssl_ca_crt.pem'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
self.assertEqual(data, out.read())
def test_ssl_ca_crt_only(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_only')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtonly.example.com',
'replication_number': '1',
'url': 'http://sslcacrtonly.example.com',
'site_url': 'http://sslcacrtonly.example.com',
'secure_access':
'https://sslcacrtonly.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.UNPROCESSABLE_ENTITY, upload.status_code)
self.assertEqual('Key incorrect', upload.text)
def test_ssl_ca_crt_garbage(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_garbage')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtgarbage.example.com',
'replication_number': '1',
'url': 'http://sslcacrtgarbage.example.com',
'site_url': 'http://sslcacrtgarbage.example.com',
'secure_access':
'https://sslcacrtgarbage.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
_, ca_key_pem, csr, _ = createCSR(
parameter_dict['domain'])
_, ca_certificate_pem = self.ca.signCSR(csr)
data = ca_certificate_pem + ca_key_pem + 'some garbage'
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
ca_certificate_pem,
der2pem(result.peercert)
)
self.assertEqualResultJson(result, 'Path', '/test-path')
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'autocert',
'_ssl_ca_crt_garbage.pem'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
self.assertEqual(data, out.read())
def test_ssl_ca_crt_does_not_match(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_does_not_match')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtdoesnotmatch.example.com',
'replication_number': '1',
'url': 'http://sslcacrtdoesnotmatch.example.com',
'site_url': 'http://sslcacrtdoesnotmatch.example.com',
'secure_access':
'https://sslcacrtdoesnotmatch.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.certificate_pem + self.key_pem + self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'autocert',
'_ssl_ca_crt_does_not_match.pem'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
self.assertEqual(data, out.read())
def test_https_only(self):
parameter_dict = self.assertSlaveBase('https-only')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
def test_custom_domain(self):
reference = 'custom_domain'
hostname = 'mycustomdomain'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_custom_domain_server_alias(self):
reference = 'custom_domain_server_alias'
hostname = 'mycustomdomainserveralias'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
result = fakeHTTPSResult(
'mycustomdomainserveralias1.example.com', parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_custom_domain_wildcard(self):
parameter_dict = self.parseSlaveParameterDict('custom_domain_wildcard')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '*.customdomain.example.com',
'replication_number': '1',
'url': 'http://*.customdomain.example.com',
'site_url': 'http://*.customdomain.example.com',
'secure_access': 'https://*.customdomain.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
'wild.customdomain.example.com', parameter_dict['public-ipv4'],
'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_custom_domain_ssl_crt_ssl_key(self):
reference = 'custom_domain_ssl_crt_ssl_key'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.customdomain_certificate_pem + \
self.customdomain_key_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.customdomain_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_zope(self):
parameter_dict = self.assertSlaveBase('type-zope')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//typezope.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://typezope.example.com:%s/test-path/deep/.././deeper' % (
HTTP_PORT,),
result.headers['Location']
)
def test_type_zope_prefer_gzip_encoding_to_backend_https_only(self):
parameter_dict = self.assertSlaveBase(
'type-zope-prefer-gzip-encoding-to-backend-https-only')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
'/VirtualHostRoot/test-path/deeper'
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
def test_type_zope_prefer_gzip_encoding_to_backend(self):
parameter_dict = self.assertSlaveBase(
'type-zope-prefer-gzip-encoding-to-backend')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackend.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deep/.././deeper' % (
parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackend.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deep/.././deeper' % (
parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
def test_type_zope_virtualhostroot_http_port(self):
parameter_dict = self.assertSlaveBase(
'type-zope-virtualhostroot-http-port')
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//typezopevirtualhostroothttpport'
'.example.com:12345//VirtualHostRoot/test-path'
)
def test_type_zope_virtualhostroot_https_port(self):
parameter_dict = self.assertSlaveBase(
'type-zope-virtualhostroot-https-port')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//typezopevirtualhostroothttpsport'
'.example.com:12345//VirtualHostRoot/test-path'
)
def test_type_notebook(self):
reference = 'type-notebook'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path',
HTTPS_PORT)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test/terminals/websocket/test',
HTTPS_PORT)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/terminals/websocket')
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_type_websocket(self):
parameter_dict = self.assertSlaveBase(
'type-websocket')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
headers={'Connection': 'Upgrade'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/test-path'
)
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertTrue('x-real-ip' in j['Incoming Headers'])
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_type_websocket_websocket_transparent_false(self):
parameter_dict = self.assertSlaveBase(
'type-websocket-websocket-transparent-false')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
headers={'Connection': 'Upgrade'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/test-path'
)
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
parsed = urlparse.urlparse(self.backend_url)
self.assertBackendHeaders(
j['Incoming Headers'], parsed.hostname, port='17', proto='irc',
ignore_header_list=['Host'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertFalse('x-real-ip' in j['Incoming Headers'])
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_type_websocket_websocket_path_list(self):
parameter_dict = self.assertSlaveBase(
'type-websocket-websocket-path-list')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
headers={'Connection': 'Upgrade'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertTrue('x-real-ip' in j['Incoming Headers'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
headers={'Connection': 'Upgrade'})
self.assertEqualResultJson(
result,
'Path',
'/ws/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertTrue('x-real-ip' in j['Incoming Headers'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'with%20space/test-path', headers={'Connection': 'Upgrade'})
self.assertEqualResultJson(
result,
'Path',
'/with%20space/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertTrue('x-real-ip' in j['Incoming Headers'])
def test_type_websocket_websocket_path_list_websocket_transparent_false(
self):
parameter_dict = self.assertSlaveBase(
'type-websocket-websocket-path-list-websocket-transparent-false')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
headers={'Connection': 'Upgrade'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
parsed = urlparse.urlparse(self.backend_url)
self.assertBackendHeaders(
j['Incoming Headers'], parsed.hostname, port='17', proto='irc',
ignore_header_list=['Host'])
self.assertFalse('x-real-ip' in j['Incoming Headers'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
headers={'Connection': 'Upgrade'})
self.assertEqualResultJson(
result,
'Path',
'/ws/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(
j['Incoming Headers'], parsed.hostname, port='17', proto='irc',
ignore_header_list=['Host'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertFalse('x-real-ip' in j['Incoming Headers'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'with%20space/test-path', headers={'Connection': 'Upgrade'})
self.assertEqualResultJson(
result,
'Path',
'/with%20space/test-path'
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(
j['Incoming Headers'], parsed.hostname, port='17', proto='irc',
ignore_header_list=['Host'])
self.assertEqual(
'Upgrade',
j['Incoming Headers']['connection']
)
self.assertFalse('x-real-ip' in j['Incoming Headers'])
@skip('Feature postponed')
def test_type_eventsource(self):
# Caddy: For event source, if I understand
# https://github.com/mholt/caddy/issues/1355 correctly, we could use
# Caddy as a proxy in front of nginx-push-stream . If we have a
# "central shared" caddy instance, can it handle keeping connections
# opens for many clients ?
parameter_dict = self.parseSlaveParameterDict('type-eventsource')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'typeeventsource.nginx.example.com',
'replication_number': '1',
'url': 'http://typeeventsource.nginx.example.com',
'site_url': 'http://typeeventsource.nginx.example.com',
'secure_access': 'https://typeeventsource.nginx.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'pub',
# NGINX_HTTPS_PORT
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
'',
result.content
)
headers = result.headers.copy()
self.assertKeyWithPop('Expires', headers)
self.assertKeyWithPop('Date', headers)
self.assertEqual(
{
'X-Nginx-PushStream-Explain': 'No channel id provided.',
'Content-Length': '0',
'Cache-Control': 'no-cache, no-store, must-revalidate',
'Connection': 'keep-alive',
'Server': 'nginx'
},
headers
)
def test_type_redirect(self):
parameter_dict = self.assertSlaveBase('type-redirect')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'%stest-path/deeper' % (self.backend_url,),
result.headers['Location']
)
def test_type_redirect_custom_domain(self):
reference = 'type-redirect-custom_domain'
hostname = 'customdomaintyperedirect'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'%stest-path/deeper' % (self.backend_url,),
result.headers['Location']
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
parameter_dict = self.parseSlaveParameterDict(
'ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslproxyverifysslproxycacrtunverified.example.com',
'replication_number': '1',
'url': 'http://sslproxyverifysslproxycacrtunverified.example.com',
'site_url':
'http://sslproxyverifysslproxycacrtunverified.example.com',
'secure_access':
'https://sslproxyverifysslproxycacrtunverified.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://sslproxyverifysslproxycacrtunverified.example.com:%s/'
'test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase('ssl-proxy-verify_ssl_proxy_ca_crt')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertFalse('Content-Encoding' in result.headers)
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://sslproxyverifysslproxycacrt.example.com:%s/test-path' % (
HTTP_PORT,),
result_http.headers['Location']
)
def test_ssl_proxy_verify_unverified(self):
parameter_dict = self.assertSlaveBase('ssl-proxy-verify-unverified')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
parameter_dict = self.parseSlaveParameterDict(
'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain':
'enablecachesslproxyverifysslproxycacrtunverified.example.com',
'replication_number': '1',
'url':
'http://enablecachesslproxyverifysslproxycacrtunverified.example.com',
'site_url':
'http://enablecachesslproxyverifysslproxycacrtunverified.example.com',
'secure_access':
'https://enablecachesslproxyverifysslproxycacrtunverified.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://enablecachesslproxyverifysslproxycacrtunverified.example.com'
':%s/test-path/deeper' % (HTTP_PORT,),
result_http.headers['Location']
)
def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase(
'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
},
headers
)
def test_enable_cache_ssl_proxy_verify_unverified(self):
parameter_dict = self.assertSlaveBase(
'enable_cache-ssl-proxy-verify-unverified')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
parameter_dict = self.parseSlaveParameterDict(
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'typezopesslproxyverifysslproxycacrtunverified.example.com',
'replication_number': '1',
'url':
'http://typezopesslproxyverifysslproxycacrtunverified.example.com',
'site_url':
'http://typezopesslproxyverifysslproxycacrtunverified.example.com',
'secure_access':
'https://typezopesslproxyverifysslproxycacrtunverified.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://typezopesslproxyverifysslproxycacrtunverified.example.com:%s/'
'test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase(
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//'
'typezopesslproxyverifysslproxycacrt.example.com:443/'
'/VirtualHostRoot/test-path'
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://typezopesslproxyverifysslproxycacrt.example.com:'
'%s/test-path' % (HTTP_PORT,),
result.headers['Location']
)
def test_type_zope_ssl_proxy_verify_unverified(self):
parameter_dict = self.assertSlaveBase(
'type-zope-ssl-proxy-verify-unverified')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.SERVICE_UNAVAILABLE,
result.status_code
)
def test_monitor_ipv6_test(self):
parameter_dict = self.assertSlaveBase('monitor-ipv6-test')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://monitoripv6test.example.com:%s/test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
'check-_monitor-ipv6-test-ipv6-packet-list-test.py'))[0]
# get promise module and check that parameters are ok
self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file),
{
'frequency': '720',
'address': 'monitor-ipv6-test'
}
)
def test_monitor_ipv4_test(self):
parameter_dict = self.assertSlaveBase('monitor-ipv4-test')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://monitoripv4test.example.com:%s/test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
'check-_monitor-ipv4-test-ipv4-packet-list-test.py'))[0]
# get promise module and check that parameters are ok
self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file),
{
'frequency': '720',
'ipv4': 'true',
'address': 'monitor-ipv4-test',
}
)
def test_ciphers(self):
parameter_dict = self.assertSlaveBase('ciphers')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://ciphers.example.com:%s/test-path' % (HTTP_PORT,),
result_http.headers['Location']
)
configuration_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'caddy-slave-conf.d', '_ciphers.conf'
))[0]
self.assertTrue(
'ciphers RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA'
in open(configuration_file).read()
)
def test_enable_cache_custom_domain(self):
reference = 'enable_cache_custom_domain'
hostname = 'customdomainenablecache'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600'
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
def test_enable_cache_server_alias(self):
parameter_dict = self.assertSlaveBase('enable_cache_server_alias')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600'
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
result = fakeHTTPResult(
'enablecacheserveralias1.example.com', parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://enablecacheserveralias1.example.com:%s/test-path/deeper' % (
HTTP_PORT,),
result.headers['Location']
)
def test_enable_cache(self):
parameter_dict = self.assertSlaveBase('enable_cache')
source_ip = '127.0.0.1'
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600',
},
source_ip=source_ip
)
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600'
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
# check stale-if-error support (assumes stale-while-revalidate is same)
# wait a bit for max-age to expire
time.sleep(2)
# real check: cache access provides old data, access cache directly, as
# caddy has to be stopped
try:
# stop the backend, to have error on while connecting to it
self.stopServerProcess()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600',
},
source_ip=source_ip
)
self.assertEqual(result.status_code, httplib.OK)
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
self.assertKeyWithPop('Expires', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
# ATS does not cache the cookied text content, see:
# https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/\
# configuration/cache-basics.en.html#caching-cookied-objects
# 'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600',
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
finally:
self.startServerProcess()
# END: check stale-if-error support
def test_enable_cache_ats_timeout(self):
parameter_dict = self.assertSlaveBase('enable_cache')
# check that timeout seen by ATS does not result in many queries done
# to the backend and that next request works like a charm
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test_enable_cache_ats_timeout', headers={
'Timeout': '15',
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
# ATS timed out
self.assertEqual(
httplib.GATEWAY_TIMEOUT,
result.status_code
)
backend_haproxy_log_file = glob.glob(
os.path.join(
self.instance_path, '*', 'var', 'log', 'backend-haproxy.log'
))[0]
matching_line_amount = 0
pattern = re.compile(
r'.* _enable_cache-http.backend .* 504 .*'
'"GET .test_enable_cache_ats_timeout HTTP.1.1"$')
with open(backend_haproxy_log_file) as fh:
for line in fh.readlines():
if pattern.match(line):
matching_line_amount += 1
# Haproxy backend received maximum one connection
self.assertIn(matching_line_amount, [0, 1])
timeout = 5
b = time.time()
# ATS created squid.log with a delay
while True:
if (time.time() - b) > timeout:
self.fail('Squid log file did not appear in %ss' % (timeout,))
ats_log_file_list = glob.glob(
os.path.join(
self.instance_path, '*', 'var', 'log', 'trafficserver', 'squid.log'
))
if len(ats_log_file_list) == 1:
ats_log_file = ats_log_file_list[0]
break
time.sleep(0.1)
pattern = re.compile(
r'.*ERR_READ_TIMEOUT/504 .*test_enable_cache_ats_timeout'
'.*TIMEOUT_DIRECT*')
timeout = 5
b = time.time()
# ATS needs some time to flush logs
while True:
matching_line_amount = 0
if (time.time() - b) > timeout:
break
with open(ats_log_file) as fh:
for line in fh.readlines():
if pattern.match(line):
matching_line_amount += 1
if matching_line_amount > 0:
break
time.sleep(0.1)
# ATS has maximum one entry for this query
self.assertIn(matching_line_amount, [0, 1])
# the result is available immediately after
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_enable_cache_disable_no_cache_request(self):
parameter_dict = self.assertSlaveBase(
'enable_cache-disable-no-cache-request')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
headers={'Pragma': 'no-cache', 'Cache-Control': 'something'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value'
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
try:
j = result.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertFalse('pragma' in j['Incoming Headers'].keys())
def test_enable_cache_disable_via_header(self):
parameter_dict = self.assertSlaveBase('enable_cache-disable-via-header')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
},
headers
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.11\)$'
)
def test_enable_http2_false(self):
parameter_dict = self.assertSlaveBase('enable-http2-false')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
# drop vary-keys
headers.pop('Content-Length', None)
headers.pop('Transfer-Encoding', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-Type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
},
headers
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_default(self):
parameter_dict = self.assertSlaveBase('enable-http2-default')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
# drop vary-keys
headers.pop('Content-Length', None)
headers.pop('Transfer-Encoding', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
},
headers
)
self.assertTrue(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_prefer_gzip_encoding_to_backend_https_only(self):
parameter_dict = self.assertSlaveBase(
'prefer-gzip-encoding-to-backend-https-only')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'deflate'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'deflate', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'],
port=HTTP_PORT, proto='http')
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'deflate'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'],
port=HTTP_PORT, proto='http')
self.assertEqual(
'deflate', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_prefer_gzip_encoding_to_backend(self):
parameter_dict = self.assertSlaveBase(
'prefer-gzip-encoding-to-backend')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'deflate'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'deflate', result.json()['Incoming Headers']['accept-encoding'])
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'gzip, deflate'})
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deeper' % (parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={'Accept-Encoding': 'deflate'})
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deeper' % (parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deeper' % (parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://%s:%s/test-path/deeper' % (parameter_dict['domain'], HTTP_PORT),
result.headers['Location']
)
def test_disabled_cookie_list(self):
parameter_dict = self.assertSlaveBase('disabled-cookie-list')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
cookies=dict(
Chocolate='absent',
Vanilia='absent',
Coffee='present'
))
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
self.assertBackendHeaders(
result.json()['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'Coffee=present', result.json()['Incoming Headers']['cookie'])
def test_https_url(self):
parameter_dict = self.assertSlaveBase('url_https-url')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/https/test-path/deeper')
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://urlhttpsurl.example.com:%s/test-path/deeper' % (HTTP_PORT,),
result_http.headers['Location']
)
# check that timeouts are correctly set in the haproxy configuration
backend_configuration_file = glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
content = fh.read()
self.assertTrue("""backend _url_https-url-http
timeout server 15s
timeout connect 10s
retries 5""" in content)
@skip('Impossible to instantiate cluster with stopped partition')
class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'-frontend-quantity': 2,
'-sla-2-computer_guid': cls.slap._computer_id,
'-frontend-2-state': 'stopped',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'replicate': {
'url': cls.backend_url,
'enable_cache': True,
},
}
def test(self):
parameter_dict = self.parseSlaveParameterDict('replicate')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'replicate.example.com',
'replication_number': '2',
'url': 'http://replicate.example.com',
'site_url': 'http://replicate.example.com',
'secure_access': 'https://replicate.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqualResultJson(result_http, 'Path', '/test-path')
# prove 2nd frontend by inspection of the instance
slave_configuration_name = '_replicate.conf'
slave_configuration_file_list = [
'/'.join([f[0], slave_configuration_name]) for f in [
q for q in os.walk(self.instance_path)
if slave_configuration_name in q[2]
]
]
self.assertEqual(
2, len(slave_configuration_file_list), slave_configuration_file_list)
@skip('Impossible to instantiate cluster with destroyed partition')
class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'-frontend-quantity': 2,
'-sla-2-computer_guid': cls.slap._computer_id,
'-frontend-2-state': 'destroyed',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'empty': {
'url': cls.backend_url,
'enable_cache': True,
}
}
def test_extra_slave_instance_list_not_present_destroyed_request(self):
buildout_file = os.path.join(
self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
with open(buildout_file) as fh:
buildout_file_content = fh.read()
node_1_present = re.search(
"^config-frontend-name = !py!'caddy-frontend-1'$",
buildout_file_content, flags=re.M) is not None
node_2_present = re.search(
"^config-frontend-name = !py!'caddy-frontend-2'$",
buildout_file_content, flags=re.M) is not None
self.assertTrue(node_1_present)
self.assertFalse(node_2_present)
class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'enable-http2-by-default': 'false',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'enable-http2-default': {
},
'enable-http2-false': {
'enable-http2': 'false',
},
'enable-http2-true': {
'enable-http2': 'true',
},
'dummy-cached': {
'url': cls.backend_url,
'enable_cache': True,
}
}
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_false(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-false')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2false.example.com',
'replication_number': '1',
'url': 'http://enablehttp2false.example.com',
'site_url': 'http://enablehttp2false.example.com',
'secure_access':
'https://enablehttp2false.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertTrue(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'enable-http2-default': {
},
'enable-http2-false': {
'enable-http2': 'false',
},
'enable-http2-true': {
'enable-http2': 'true',
},
'dummy-cached': {
'url': cls.backend_url,
'enable_cache': True,
}
}
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertTrue(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_false(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-false')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2false.example.com',
'replication_number': '1',
'url': 'http://enablehttp2false.example.com',
'site_url': 'http://enablehttp2false.example.com',
'secure_access':
'https://enablehttp2false.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'public-ipv4': self._ipv4_address,
},
parameter_dict
)
self.assertTrue(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'default': {
'url': cls.backend_url,
'enable_cache': True
},
}
@classmethod
def waitForSlave(cls):
# no need to wait for slave availability here
return True
def test_default(self):
parameter_dict = self.parseSlaveParameterDict('default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'default.None',
'replication_number': '1',
'url': 'http://default.None',
'site_url': 'http://default.None',
'secure_access': 'https://default.None',
'public-ipv4': 'None',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
re6st_connectivity_promise_list = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
're6st-connectivity.py'))
self.assertEqual(1, len(re6st_connectivity_promise_list))
re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
self.assertEqual(
getPluginParameterDict(
self.software_path, re6st_connectivity_promise_file),
{
'url': 'http://[2001:67c:1254:4::1]/index.html',
}
)
@skip('New test system cannot be used with failing promises')
class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
're6st-verification-url': 'some-re6st-verification-url',
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'default': {
'url': cls.backend_url,
'enable_cache': True,
},
}
def test_default(self):
parameter_dict = self.parseSlaveParameterDict('default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'default.None',
'replication_number': '1',
'url': 'http://default.None',
'site_url': 'http://default.None',
'secure_access': 'https://default.None',
'public-ipv4': 'None',
},
parameter_dict
)
re6st_connectivity_promise_list = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
're6st-connectivity.py'))
self.assertEqual(1, len(re6st_connectivity_promise_list))
re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
self.assertEqual(
getPluginParameterDict(
self.software_path, re6st_connectivity_promise_file),
{
'url': 'some-re6st-verification-url',
}
)
@skip('Impossible to instantiate cluster with stopped partition')
class TestDefaultMonitorHttpdPort(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'-frontend-1-state': 'stopped',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def runKedifaUpdater(cls):
return
@classmethod
def getSlaveParameterDictDict(cls):
return {
'test': {
'url': cls.backend_url,
},
'testcached': {
'url': cls.backend_url,
'enable_cache': True,
},
}
def test(self):
parameter_dict = self.parseSlaveParameterDict('test')
self.assertKeyWithPop('log-access-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'test.None', 'replication_number': '1',
'url': 'http://test.None', 'site_url': 'http://test.None',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'secure_access': 'https://test.None', 'public-ipv4': 'None'},
parameter_dict
)
master_monitor_conf = open(os.path.join(
self.instance_path, 'T-0', 'etc',
'monitor-httpd.conf')).read()
slave_monitor_conf = open(os.path.join(
self.instance_path, 'T-2', 'etc',
'monitor-httpd.conf')).read()
self.assertTrue(
'Listen [%s]:8196' % (self._ipv6_address,) in master_monitor_conf)
self.assertTrue(
'Listen [%s]:8072' % (self._ipv6_address,) in slave_monitor_conf)
class TestSlaveGlobalDisableHttp2(TestSlave):
@classmethod
def getInstanceParameterDict(cls):
instance_parameter_dict = super(
TestSlaveGlobalDisableHttp2, cls).getInstanceParameterDict()
instance_parameter_dict['global-disable-http2'] = 'TrUe'
return instance_parameter_dict
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
# drop vary-keys
headers.pop('Content-Length', None)
headers.pop('Transfer-Encoding', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
},
headers
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
class TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2(
TestEnableHttp2ByDefaultFalseSlave):
@classmethod
def getInstanceParameterDict(cls):
instance_parameter_dict = super(
TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2,
cls).getInstanceParameterDict()
instance_parameter_dict['global-disable-http2'] = 'TrUe'
return instance_parameter_dict
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
class TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2(
TestEnableHttp2ByDefaultDefaultSlave):
@classmethod
def getInstanceParameterDict(cls):
instance_parameter_dict = super(
TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2,
cls).getInstanceParameterDict()
instance_parameter_dict['global-disable-http2'] = 'TrUe'
return instance_parameter_dict
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertFalse(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def setUpMaster(cls):
# run partition until AIKC finishes
cls.runComputerPartitionUntil(
cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
ca_certificate = requests.get(
parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
# Do not upload certificates for the master partition
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'apache-certificate': cls.certificate_pem,
'apache-key': cls.key_pem,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'ssl_from_master_kedifa_overrides_master_certificate': {
'url': cls.backend_url,
'enable_cache': True
},
}
def test_ssl_from_master_kedifa_overrides_master_certificate(self):
reference = 'ssl_from_master_kedifa_overrides_master_certificate'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = \
createSelfSignedCertificate([parameter_dict['domain']])
master_parameter_dict = \
self.requestDefaultInstance().getConnectionParameterDict()
auth = requests.get(
master_parameter_dict['master-key-generate-auth-url'],
verify=self.ca_certificate_file)
requests.put(
master_parameter_dict['master-key-upload-url'] + auth.text,
data=key_pem + certificate_pem,
verify=self.ca_certificate_file)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
class TestSlaveSlapOSMasterCertificateCompatibility(
SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def setUpMaster(cls):
# run partition until AIKC finishes
cls.runComputerPartitionUntil(
cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
ca_certificate = requests.get(
parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
# Do not upload certificates for the master partition
@classmethod
def prepareCertificate(cls):
_, cls.ssl_from_slave_key_pem, _, cls.ssl_from_slave_certificate_pem = \
createSelfSignedCertificate(
[
'sslfromslave.example.com',
])
_, cls.ssl_from_slave_kedifa_overrides_key_pem, _, \
cls.ssl_from_slave_kedifa_overrides_certificate_pem = \
createSelfSignedCertificate(
[
'sslfromslavekedifaoverrides.example.com',
])
_, cls.type_notebook_ssl_from_slave_key_pem, _, \
cls.type_notebook_ssl_from_slave_certificate_pem = \
createSelfSignedCertificate(
[
'typenotebooksslfromslave.example.com',
])
_, cls.type_notebook_ssl_from_slave_kedifa_overrides_key_pem, _, \
cls.type_notebook_ssl_from_slave_kedifa_overrides_certificate_pem = \
createSelfSignedCertificate(
[
'typenotebooksslfromslavekedifaoverrides.example.com',
])
cls.ca = CertificateAuthority(
'TestSlaveSlapOSMasterCertificateCompatibility')
_, cls.customdomain_ca_key_pem, csr, _ = createCSR(
'customdomainsslcrtsslkeysslcacrt.example.com')
_, cls.customdomain_ca_certificate_pem = cls.ca.signCSR(csr)
_, cls.sslcacrtgarbage_ca_key_pem, csr, _ = createCSR(
'sslcacrtgarbage.example.com')
_, cls.sslcacrtgarbage_ca_certificate_pem = cls.ca.signCSR(csr)
_, cls.ssl_from_slave_ca_key_pem, csr, _ = createCSR(
'sslfromslave.example.com')
_, cls.ssl_from_slave_ca_certificate_pem = cls.ca.signCSR(csr)
_, cls.customdomain_key_pem, _, cls.customdomain_certificate_pem = \
createSelfSignedCertificate(['customdomainsslcrtsslkey.example.com'])
super(
TestSlaveSlapOSMasterCertificateCompatibility, cls).prepareCertificate()
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'apache-certificate': cls.certificate_pem,
'apache-key': cls.key_pem,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'ssl_from_master': {
'url': cls.backend_url,
'enable_cache': True,
},
'ssl_from_master_kedifa_overrides': {
'url': cls.backend_url,
},
'ssl_from_slave': {
'url': cls.backend_url,
'ssl_crt': cls.ssl_from_slave_certificate_pem,
'ssl_key': cls.ssl_from_slave_key_pem,
},
'ssl_from_slave_kedifa_overrides': {
'url': cls.backend_url,
'ssl_crt': cls.ssl_from_slave_kedifa_overrides_certificate_pem,
'ssl_key': cls.ssl_from_slave_kedifa_overrides_key_pem,
},
'custom_domain_ssl_crt_ssl_key': {
'url': cls.backend_url,
'ssl_crt': cls.customdomain_certificate_pem,
'ssl_key': cls.customdomain_key_pem,
'custom_domain': 'customdomainsslcrtsslkey.example.com'
},
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt': {
'url': cls.backend_url,
'ssl_crt': cls.customdomain_ca_certificate_pem,
'ssl_key': cls.customdomain_ca_key_pem,
'ssl_ca_crt': cls.ca.certificate_pem,
'custom_domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
},
'ssl_ca_crt_garbage': {
'url': cls.backend_url,
'ssl_crt': cls.sslcacrtgarbage_ca_certificate_pem,
'ssl_key': cls.sslcacrtgarbage_ca_key_pem,
'ssl_ca_crt': 'some garbage',
},
'ssl_ca_crt_does_not_match': {
'url': cls.backend_url,
'ssl_crt': cls.certificate_pem,
'ssl_key': cls.key_pem,
'ssl_ca_crt': cls.ca.certificate_pem,
},
'type-notebook-ssl_from_master': {
'url': cls.backend_url,
'type': 'notebook',
},
'type-notebook-ssl_from_slave': {
'url': cls.backend_url,
'ssl_crt': cls.type_notebook_ssl_from_slave_certificate_pem,
'ssl_key': cls.type_notebook_ssl_from_slave_key_pem,
'type': 'notebook',
},
'type-notebook-ssl_from_master_kedifa_overrides': {
'url': cls.backend_url,
'type': 'notebook',
},
'type-notebook-ssl_from_slave_kedifa_overrides': {
'url': cls.backend_url,
'ssl_crt':
cls.type_notebook_ssl_from_slave_kedifa_overrides_certificate_pem,
'ssl_key':
cls.type_notebook_ssl_from_slave_kedifa_overrides_key_pem,
'type': 'notebook',
}
}
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '12',
'rejected-slave-amount': '0',
'slave-amount': '12',
'rejected-slave-dict': {
# u"_ssl_ca_crt_only":
# [u"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
# u"_ssl_key-ssl_crt-unsafe":
# [u"slave ssl_key and ssl_crt does not match"]
},
'warning-list': [
u'apache-certificate is obsolete, please use master-key-upload-url',
u'apache-key is obsolete, please use master-key-upload-url',
],
'warning-slave-dict': {
u'_custom_domain_ssl_crt_ssl_key': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url'
],
u'_custom_domain_ssl_crt_ssl_key_ssl_ca_crt': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
u'ssl_ca_crt is obsolete, please use key-upload-url'
],
u'_ssl_ca_crt_does_not_match': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
u'ssl_ca_crt is obsolete, please use key-upload-url',
],
u'_ssl_ca_crt_garbage': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
u'ssl_ca_crt is obsolete, please use key-upload-url',
],
# u'_ssl_ca_crt_only': [
# u'ssl_ca_crt is obsolete, please use key-upload-url',
# ],
u'_ssl_from_slave': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
],
u'_ssl_from_slave_kedifa_overrides': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
],
# u'_ssl_key-ssl_crt-unsafe': [
# u'ssl_key is obsolete, please use key-upload-url',
# u'ssl_crt is obsolete, please use key-upload-url',
# ],
u'_type-notebook-ssl_from_slave': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
],
u'_type-notebook-ssl_from_slave_kedifa_overrides': [
u'ssl_key is obsolete, please use key-upload-url',
u'ssl_crt is obsolete, please use key-upload-url',
],
}
}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_ssl_from_master(self):
parameter_dict = self.parseSlaveParameterDict('ssl_from_master')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = 'ssl_from_master'.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_master_kedifa_overrides(self):
reference = 'ssl_from_master_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = \
createSelfSignedCertificate([parameter_dict['domain']])
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_slave(self):
reference = 'ssl_from_slave'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.ssl_from_slave_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_slave_kedifa_overrides(self):
reference = 'ssl_from_slave_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.ssl_from_slave_kedifa_overrides_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = \
createSelfSignedCertificate([parameter_dict['domain']])
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_master(self):
reference = 'type-notebook-ssl_from_master'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_master_kedifa_overrides(self):
reference = 'type-notebook-ssl_from_master_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = \
createSelfSignedCertificate([parameter_dict['domain']])
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_slave(self):
reference = 'type-notebook-ssl_from_slave'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
self.type_notebook_ssl_from_slave_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_slave_kedifa_overrides(self):
reference = 'type-notebook-ssl_from_slave_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
self.type_notebook_ssl_from_slave_kedifa_overrides_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = \
createSelfSignedCertificate([parameter_dict['domain']])
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
data=data,
verify=self.ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
HTTPS_PORT)
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
@skip('Not implemented in new test system')
def test_custom_domain_ssl_crt_ssl_key(self):
reference = 'custom_domain_ssl_crt_ssl_key'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': ['ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url']
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.customdomain_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_ca_crt(self):
parameter_dict = self.parseSlaveParameterDict(
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
'replication_number': '1',
'url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'site_url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'secure_access':
'https://customdomainsslcrtsslkeysslcacrt.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_ca_crt is obsolete, please use key-upload-url'
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.customdomain_ca_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'bbb-ssl',
'_custom_domain_ssl_crt_ssl_key_ssl_ca_crt.crt'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
expected = self.customdomain_ca_certificate_pem + '\n' + \
self.ca.certificate_pem + '\n' + self.customdomain_ca_key_pem
self.assertEqual(
expected,
out.read()
)
ca = CertificateAuthority(
'TestSlaveSlapOSMasterCertificateCompatibility')
_, customdomain_ca_key_pem, csr, _ = createCSR(
'customdomainsslcrtsslkeysslcacrt.example.com')
_, customdomain_ca_certificate_pem = ca.signCSR(csr)
slave_parameter_dict = self.getSlaveParameterDictDict()[
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt'].copy()
slave_parameter_dict.update(
ssl_crt=customdomain_ca_certificate_pem,
ssl_key=customdomain_ca_key_pem,
ssl_ca_crt=ca.certificate_pem,
)
self.slap.request(
software_release=self.getSoftwareURL(),
partition_reference='custom_domain_ssl_crt_ssl_key_ssl_ca_crt',
partition_parameter_kw=slave_parameter_dict,
shared=True
)
self.slap.waitForInstance()
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
customdomain_ca_certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'bbb-ssl',
'_custom_domain_ssl_crt_ssl_key_ssl_ca_crt.crt'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
expected = customdomain_ca_certificate_pem + '\n' + ca.certificate_pem \
+ '\n' + customdomain_ca_key_pem
self.assertEqual(
expected,
out.read()
)
def test_ssl_ca_crt_garbage(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_garbage')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtgarbage.example.com',
'replication_number': '1',
'url': 'http://sslcacrtgarbage.example.com',
'site_url': 'http://sslcacrtgarbage.example.com',
'secure_access':
'https://sslcacrtgarbage.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_ca_crt is obsolete, please use key-upload-url']
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.sslcacrtgarbage_ca_certificate_pem,
der2pem(result.peercert)
)
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_ca_crt_does_not_match(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_does_not_match')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtdoesnotmatch.example.com',
'replication_number': '1',
'url': 'http://sslcacrtdoesnotmatch.example.com',
'site_url': 'http://sslcacrtdoesnotmatch.example.com',
'secure_access':
'https://sslcacrtdoesnotmatch.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_ca_crt is obsolete, please use key-upload-url'
]
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
certificate_file_list = glob.glob(os.path.join(
self.instance_path, '*', 'srv', 'bbb-ssl',
'_ssl_ca_crt_does_not_match.crt'))
self.assertEqual(1, len(certificate_file_list))
certificate_file = certificate_file_list[0]
with open(certificate_file) as out:
expected = self.certificate_pem + '\n' + self.ca.certificate_pem + \
'\n' + self.key_pem
self.assertEqual(
expected,
out.read()
)
class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def setUpMaster(cls):
# run partition until AIKC finishes
cls.runComputerPartitionUntil(
cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
ca_certificate = requests.get(
parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
assert ca_certificate.status_code == httplib.OK
cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
# Do not upload certificates for the master partition
instance_parameter_dict = {
'domain': 'example.com',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
}
@classmethod
def getInstanceParameterDict(cls):
if 'apache-certificate' not in cls.instance_parameter_dict:
cls.instance_parameter_dict.update(**{
'apache-certificate': cls.certificate_pem,
'apache-key': cls.key_pem,
})
cls.instance_parameter_dict['public-ipv4'] = cls._ipv4_address
return cls.instance_parameter_dict
@classmethod
def getSlaveParameterDictDict(cls):
return {
'ssl_from_master': {
'url': cls.backend_url,
'enable_cache': True,
},
}
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '1',
'rejected-slave-amount': '0',
'rejected-slave-dict': {},
'slave-amount': '1',
'warning-list': [
u'apache-certificate is obsolete, please use master-key-upload-url',
u'apache-key is obsolete, please use master-key-upload-url',
],
}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_apache_key_apache_certificate_update(self):
parameter_dict = self.parseSlaveParameterDict('ssl_from_master')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = 'ssl_from_master'.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
_, key_pem, _, certificate_pem = createSelfSignedCertificate(
[
'*.customdomain.example.com',
'*.example.com',
'*.alias1.example.com',
])
self.instance_parameter_dict.update(**{
'apache-certificate': certificate_pem,
'apache-key': key_pem,
})
self.requestDefaultInstance()
self.slap.waitForInstance()
self.runKedifaUpdater()
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
'ciphers': 'ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384'
}
@classmethod
def getSlaveParameterDictDict(cls):
return {
'default_ciphers': {
'url': cls.backend_url,
'enable_cache': True,
},
'own_ciphers': {
'ciphers': 'ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256',
'url': cls.backend_url,
'enable_cache': True,
},
}
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '2',
'rejected-slave-amount': '0',
'slave-amount': '2',
'rejected-slave-dict': {}
}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_default_ciphers(self):
parameter_dict = self.assertSlaveBase('default_ciphers')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.OK, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.FOUND, result_http.status_code)
configuration_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
'_default_ciphers.conf'
))[0]
self.assertTrue(
'ciphers ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384'
in open(configuration_file).read()
)
def test_own_ciphers(self):
parameter_dict = self.assertSlaveBase('own_ciphers')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.OK, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.FOUND, result_http.status_code)
configuration_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
'_own_ciphers.conf'
))[0]
self.assertTrue(
'ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256'
in open(configuration_file).read()
)
class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
@classmethod
def prepareCertificate(cls):
cls.ca = CertificateAuthority('TestSlaveRejectReportUnsafeDamaged')
super(TestSlaveRejectReportUnsafeDamaged, cls).prepareCertificate()
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
}
@classmethod
def setUpClass(cls):
super(TestSlaveRejectReportUnsafeDamaged, cls).setUpClass()
cls.fillSlaveParameterDictDict()
cls.requestSlaves()
try:
cls.slap.waitForInstance(
max_retry=2 # two runs shall be enough
)
except Exception:
# ignores exceptions, as problems are tested
pass
cls.updateSlaveConnectionParameterDictDict()
slave_parameter_dict_dict = {}
@classmethod
def getSlaveParameterDictDict(cls):
return cls.slave_parameter_dict_dict
@classmethod
def fillSlaveParameterDictDict(cls):
cls.slave_parameter_dict_dict = {
'url': {
'url': "https://[fd46::c2ae]:!py!u'123123'",
},
'https-url': {
'https-url': "https://[fd46::c2ae]:!py!u'123123'",
},
'ssl-proxy-verify_ssl_proxy_ca_crt_damaged': {
'url': cls.backend_https_url,
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': 'damaged',
},
'ssl-proxy-verify_ssl_proxy_ca_crt_empty': {
'url': cls.backend_https_url,
'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': '',
},
'bad-backend': {
'url': 'http://1:2:3:4',
'https-url': 'http://host.domain:badport',
},
'empty-backend': {
'url': '',
'https-url': '',
},
'custom_domain-unsafe': {
'custom_domain': '${section:option} afterspace\nafternewline',
},
'server-alias-unsafe': {
'server-alias': '${section:option} afterspace',
},
'server-alias-same': {
'url': cls.backend_url,
'server-alias': 'serveraliassame.example.com',
},
'virtualhostroot-http-port-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'virtualhostroot-http-port': '${section:option}',
},
'virtualhostroot-https-port-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'virtualhostroot-https-port': '${section:option}',
},
'default-path-unsafe': {
'type': 'zope',
'url': cls.backend_url,
'default-path': '${section:option}\nn"\newline\n}\n}proxy\n/slashed',
},
'monitor-ipv4-test-unsafe': {
'monitor-ipv4-test': '${section:option}\nafternewline ipv4',
},
'monitor-ipv6-test-unsafe': {
'monitor-ipv6-test': '${section:option}\nafternewline ipv6',
},
'bad-ciphers': {
'ciphers': 'bad ECDHE-ECDSA-AES256-GCM-SHA384 again',
},
'site_1': {
'custom_domain': 'duplicate.example.com',
},
'site_2': {
'custom_domain': 'duplicate.example.com',
},
'site_3': {
'server-alias': 'duplicate.example.com',
},
'site_4': {
'custom_domain': 'duplicate.example.com',
'server-alias': 'duplicate.example.com',
},
'ssl_ca_crt_only': {
'url': cls.backend_url,
'ssl_ca_crt': cls.ca.certificate_pem,
},
'ssl_key-ssl_crt-unsafe': {
'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
},
}
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '7',
'rejected-slave-amount': '14',
'slave-amount': '21',
'rejected-slave-dict': {
'_https-url': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
' invalid'],
'_url': [u'slave url "https://[fd46::c2ae]:!py!u\'123123\'" invalid'],
'_ssl-proxy-verify_ssl_proxy_ca_crt_damaged': [
'ssl_proxy_ca_crt is invalid'
],
'_ssl-proxy-verify_ssl_proxy_ca_crt_empty': [
'ssl_proxy_ca_crt is invalid'
],
'_bad-ciphers': [
"Cipher 'bad' is not supported.",
"Cipher 'again' is not supported."
],
'_custom_domain-unsafe': [
"custom_domain '${section:option} afterspace\\nafternewline' invalid"
],
'_server-alias-unsafe': [
"server-alias '${section:option}' not valid",
"server-alias 'afterspace' not valid"
],
'_site_2': ["custom_domain 'duplicate.example.com' clashes"],
'_site_3': ["server-alias 'duplicate.example.com' clashes"],
'_site_4': ["custom_domain 'duplicate.example.com' clashes"],
'_ssl_ca_crt_only': [
"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
'_ssl_key-ssl_crt-unsafe': [
"slave ssl_key and ssl_crt does not match"],
'_bad-backend': [
"slave url 'http://1:2:3:4' invalid",
"slave https-url 'http://host.domain:badport' invalid"],
'_empty-backend': [
"slave url '' invalid",
"slave https-url '' invalid"],
},
'warning-slave-dict': {
'_ssl_ca_crt_only': [
'ssl_ca_crt is obsolete, please use key-upload-url'],
'_ssl_key-ssl_crt-unsafe': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url']}
}
self.assertEqual(
expected_parameter_dict,
parameter_dict
)
def test_url(self):
parameter_dict = self.parseSlaveParameterDict('url')
self.assertEqual(
{
'request-error-list': [
"slave url \"https://[fd46::c2ae]:!py!u'123123'\" invalid"]
},
parameter_dict
)
def test_https_url(self):
parameter_dict = self.parseSlaveParameterDict('https-url')
self.assertEqual(
{
'request-error-list': [
"slave https-url \"https://[fd46::c2ae]:!py!u'123123'\" invalid"]
},
parameter_dict
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt_damaged(self):
parameter_dict = self.parseSlaveParameterDict(
'ssl-proxy-verify_ssl_proxy_ca_crt_damaged')
self.assertEqual(
{'request-error-list': ["ssl_proxy_ca_crt is invalid"]},
parameter_dict
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt_empty(self):
parameter_dict = self.parseSlaveParameterDict(
'ssl-proxy-verify_ssl_proxy_ca_crt_empty')
self.assertEqual(
{'request-error-list': ["ssl_proxy_ca_crt is invalid"]},
parameter_dict
)
def test_server_alias_same(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-same')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliassame.example.com',
'replication_number': '1',
'url': 'http://serveraliassame.example.com',
'site_url': 'http://serveraliassame.example.com',
'secure_access': 'https://serveraliassame.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_custom_domain_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('custom_domain-unsafe')
self.assertEqual(
{
'request-error-list': [
"custom_domain '${section:option} afterspace\\nafternewline' invalid"
]
},
parameter_dict
)
def test_server_alias_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-unsafe')
self.assertEqual(
{
'request-error-list': [
"server-alias '${section:option}' not valid", "server-alias "
"'afterspace' not valid"]
},
parameter_dict
)
def test_bad_ciphers(self):
parameter_dict = self.parseSlaveParameterDict('bad-ciphers')
self.assertEqual(
{
'request-error-list': [
"Cipher 'bad' is not supported.",
"Cipher 'again' is not supported."
]
},
parameter_dict
)
def test_virtualhostroot_http_port_unsafe(self):
parameter_dict = self.parseSlaveParameterDict(
'virtualhostroot-http-port-unsafe')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'virtualhostroothttpportunsafe.example.com',
'replication_number': '1',
'url': 'http://virtualhostroothttpportunsafe.example.com',
'site_url': 'http://virtualhostroothttpportunsafe.example.com',
'secure_access':
'https://virtualhostroothttpportunsafe.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.FOUND, result.status_code)
def test_virtualhostroot_https_port_unsafe(self):
parameter_dict = self.parseSlaveParameterDict(
'virtualhostroot-https-port-unsafe')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'virtualhostroothttpsportunsafe.example.com',
'replication_number': '1',
'url': 'http://virtualhostroothttpsportunsafe.example.com',
'site_url': 'http://virtualhostroothttpsportunsafe.example.com',
'secure_access':
'https://virtualhostroothttpsportunsafe.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/https//virtualhostroothttpsportunsafe'
'.example.com:0//VirtualHostRoot/test-path'
)
def default_path_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('default-path-unsafe')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertEqual(
{
'domain': 'defaultpathunsafe.example.com',
'replication_number': '1',
'url': 'http://defaultpathunsafe.example.com',
'site_url': 'http://defaultpathunsafe.example.com',
'secure_access': 'https://defaultpathunsafe.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], '')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(
httplib.MOVED_PERMANENTLY,
result.status_code
)
self.assertEqual(
'https://defaultpathunsafe.example.com:%s/%%24%%7Bsection%%3Aoption%%7D'
'%%0An%%22%%0Aewline%%0A%%7D%%0A%%7Dproxy%%0A/slashed' % (HTTPS_PORT,),
result.headers['Location']
)
def test_monitor_ipv4_test_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('monitor-ipv4-test-unsafe')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'monitoripv4testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv4testunsafe.example.com',
'site_url': 'http://monitoripv4testunsafe.example.com',
'secure_access': 'https://monitoripv4testunsafe.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.FOUND, result_http.status_code)
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
'check-_monitor-ipv4-test-unsafe-ipv4-packet-list-test.py'))[0]
# get promise module and check that parameters are ok
self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file),
{
'frequency': '720',
'ipv4': 'true',
'address': '${section:option}\nafternewline ipv4',
}
)
def test_monitor_ipv6_test_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('monitor-ipv6-test-unsafe')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'monitoripv6testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv6testunsafe.example.com',
'site_url': 'http://monitoripv6testunsafe.example.com',
'secure_access': 'https://monitoripv6testunsafe.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqual(httplib.SERVICE_UNAVAILABLE, result.status_code)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.FOUND, result_http.status_code)
monitor_file = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'plugin',
'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test.py'))[0]
# get promise module and check that parameters are ok
self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file),
{
'frequency': '720',
'address': '${section:option}\nafternewline ipv6'
}
)
def test_site_1(self):
parameter_dict = self.parseSlaveParameterDict('site_1')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'duplicate.example.com',
'replication_number': '1',
'url': 'http://duplicate.example.com',
'site_url': 'http://duplicate.example.com',
'secure_access': 'https://duplicate.example.com',
'public-ipv4': self._ipv4_address,
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
def test_site_2(self):
parameter_dict = self.parseSlaveParameterDict('site_2')
self.assertEqual(
{
'request-error-list': ["custom_domain 'duplicate.example.com' clashes"]
},
parameter_dict
)
def test_site_3(self):
parameter_dict = self.parseSlaveParameterDict('site_3')
self.assertEqual(
{
'request-error-list': ["server-alias 'duplicate.example.com' clashes"]
},
parameter_dict,
)
def test_site_4(self):
parameter_dict = self.parseSlaveParameterDict('site_4')
self.assertEqual(
{
'request-error-list': ["custom_domain 'duplicate.example.com' clashes"]
},
parameter_dict
)
def test_ssl_ca_crt_only(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_only')
self.assertEqual(
parameter_dict,
{
'request-error-list': [
"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
'warning-list': [
'ssl_ca_crt is obsolete, please use key-upload-url',
],
}
)
def test_ssl_key_ssl_crt_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('ssl_key-ssl_crt-unsafe')
self.assertEqual(
{
'request-error-list': ["slave ssl_key and ssl_crt does not match"],
'warning-list': [
'ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url']
},
parameter_dict
)
def test_bad_backend(self):
parameter_dict = self.parseSlaveParameterDict('bad-backend')
self.assertEqual(
{
'request-error-list': [
"slave url 'http://1:2:3:4' invalid",
"slave https-url 'http://host.domain:badport' invalid"],
},
parameter_dict
)
def test_empty_backend(self):
parameter_dict = self.parseSlaveParameterDict('empty-backend')
self.assertEqual(
{
'request-error-list': [
"slave url '' invalid",
"slave https-url '' invalid"],
},
parameter_dict
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data/ 0000775 0000000 0000000 00000000000 13712426147 0031043 5 ustar 00root root 0000000 0000000 test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0047570 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000002106 13712426147 0046245 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0046765 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0046270 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0047054 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0053117 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000002106 13712426147 0051574 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0052314 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0051617 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0052403 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0047236 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000002106 13712426147 0045713 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0046433 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0045736 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0046522 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0052565 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000002106 13712426147 0051242 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0051762 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0051265 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0052051 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestMasterRequest.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0044164 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestMasterRequest.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001011 13712426147 0042633 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestMasterRequest.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0043361 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestMasterRequest.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0042664 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestMasterRequest.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0043450 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestMasterRequestDomain.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0045314 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestMasterRequestDomain.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001011 13712426147 0043763 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0044511 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestMasterRequestDomain.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0044014 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0044600 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestRe6stVerificationUrlDefaultSlave.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0047711 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001124 13712426147 0046365 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_default_access_log
T-2/var/log/httpd/_default_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0047106 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestRe6stVerificationUrlDefaultSlave.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0046411 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0047175 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlave.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0042432 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlave.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000023626 13712426147 0041121 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_Url_access_log
T-2/var/log/httpd/_Url_backend_log
T-2/var/log/httpd/_Url_error_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_access_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_backend_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_error_log
T-2/var/log/httpd/_auth-to-backend-not-configured_access_log
T-2/var/log/httpd/_auth-to-backend-not-configured_backend_log
T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_backend_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
T-2/var/log/httpd/_custom_domain_backend_log
T-2/var/log/httpd/_custom_domain_error_log
T-2/var/log/httpd/_custom_domain_server_alias_access_log
T-2/var/log/httpd/_custom_domain_server_alias_backend_log
T-2/var/log/httpd/_custom_domain_server_alias_error_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_error_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_error_log
T-2/var/log/httpd/_custom_domain_wildcard_access_log
T-2/var/log/httpd/_custom_domain_wildcard_backend_log
T-2/var/log/httpd/_custom_domain_wildcard_error_log
T-2/var/log/httpd/_disabled-cookie-list_access_log
T-2/var/log/httpd/_disabled-cookie-list_backend_log
T-2/var/log/httpd/_disabled-cookie-list_error_log
T-2/var/log/httpd/_empty_access_log
T-2/var/log/httpd/_empty_backend_log
T-2/var/log/httpd/_empty_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_access_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_backend_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_enable_cache_access_log
T-2/var/log/httpd/_enable_cache_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_access_log
T-2/var/log/httpd/_enable_cache_custom_domain_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_error_log
T-2/var/log/httpd/_enable_cache_error_log
T-2/var/log/httpd/_enable_cache_server_alias_access_log
T-2/var/log/httpd/_enable_cache_server_alias_backend_log
T-2/var/log/httpd/_enable_cache_server_alias_error_log
T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_backend_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
T-2/var/log/httpd/_monitor-ipv6-test_backend_log
T-2/var/log/httpd/_monitor-ipv6-test_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_backend_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_error_log
T-2/var/log/httpd/_server-alias-duplicated_access_log
T-2/var/log/httpd/_server-alias-duplicated_backend_log
T-2/var/log/httpd/_server-alias-duplicated_error_log
T-2/var/log/httpd/_server-alias-empty_access_log
T-2/var/log/httpd/_server-alias-empty_backend_log
T-2/var/log/httpd/_server-alias-empty_error_log
T-2/var/log/httpd/_server-alias-wildcard_access_log
T-2/var/log/httpd/_server-alias-wildcard_backend_log
T-2/var/log/httpd/_server-alias-wildcard_error_log
T-2/var/log/httpd/_server-alias_access_log
T-2/var/log/httpd/_server-alias_backend_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_access_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_backend_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_error_log
T-2/var/log/httpd/_server-alias_error_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_access_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_backend_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_error_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_access_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_backend_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_error_log
T-2/var/log/httpd/_ssl_ca_crt_only_access_log
T-2/var/log/httpd/_ssl_ca_crt_only_backend_log
T-2/var/log/httpd/_ssl_ca_crt_only_error_log
T-2/var/log/httpd/_type-notebook_access_log
T-2/var/log/httpd/_type-notebook_backend_log
T-2/var/log/httpd/_type-notebook_error_log
T-2/var/log/httpd/_type-redirect-custom_domain_access_log
T-2/var/log/httpd/_type-redirect-custom_domain_error_log
T-2/var/log/httpd/_type-redirect_access_log
T-2/var/log/httpd/_type-redirect_error_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_backend_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_backend_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_backend_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
T-2/var/log/httpd/_type-websocket_access_log
T-2/var/log/httpd/_type-websocket_backend_log
T-2/var/log/httpd/_type-websocket_error_log
T-2/var/log/httpd/_type-zope-default-path_access_log
T-2/var/log/httpd/_type-zope-default-path_backend_log
T-2/var/log/httpd/_type-zope-default-path_error_log
T-2/var/log/httpd/_type-zope-path_access_log
T-2/var/log/httpd/_type-zope-path_backend_log
T-2/var/log/httpd/_type-zope-path_error_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_backend_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_error_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_access_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_backend_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_access_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_backend_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_error_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_access_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_backend_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
T-2/var/log/httpd/_type-zope_access_log
T-2/var/log/httpd/_type-zope_backend_log
T-2/var/log/httpd/_type-zope_error_log
T-2/var/log/httpd/_url_https-url_access_log
T-2/var/log/httpd/_url_https-url_backend_log
T-2/var/log/httpd/_url_https-url_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlave.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003411 13712426147 0041624 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-_monitor-ipv4-test-ipv4-packet-list-test.py
T-2/etc/plugin/check-_monitor-ipv6-test-ipv6-packet-list-test.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlave.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0041132 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlave.test_master_partition_state-CADDY.txt 0000664 0000000 0000000 00000001302 13712426147 0043055 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data SetEnvIf Origin "^http(s)?://(.+\.)?(monitor\.app\.officejs\.com)$" ORIGIN_DOMAIN=$0
Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Credentials "true" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" env=ORIGIN_DOMAIN
test.TestSlave.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0041716 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlaveCiphers.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0043750 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlaveCiphers.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001421 13712426147 0042424 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_default_ciphers_access_log
T-2/var/log/httpd/_default_ciphers_backend_log
T-2/var/log/httpd/_default_ciphers_error_log
T-2/var/log/httpd/_own_ciphers_access_log
T-2/var/log/httpd/_own_ciphers_backend_log
T-2/var/log/httpd/_own_ciphers_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0043145 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlaveCiphers.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0042450 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlaveCiphers.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0043234 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlaveGlobalDisableHttp2.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0045761 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000023626 13712426147 0044450 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_Url_access_log
T-2/var/log/httpd/_Url_backend_log
T-2/var/log/httpd/_Url_error_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_access_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_backend_log
T-2/var/log/httpd/_auth-to-backend-backend-ignore_error_log
T-2/var/log/httpd/_auth-to-backend-not-configured_access_log
T-2/var/log/httpd/_auth-to-backend-not-configured_backend_log
T-2/var/log/httpd/_auth-to-backend-not-configured_error_log
T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_backend_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
T-2/var/log/httpd/_custom_domain_backend_log
T-2/var/log/httpd/_custom_domain_error_log
T-2/var/log/httpd/_custom_domain_server_alias_access_log
T-2/var/log/httpd/_custom_domain_server_alias_backend_log
T-2/var/log/httpd/_custom_domain_server_alias_error_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_error_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_error_log
T-2/var/log/httpd/_custom_domain_wildcard_access_log
T-2/var/log/httpd/_custom_domain_wildcard_backend_log
T-2/var/log/httpd/_custom_domain_wildcard_error_log
T-2/var/log/httpd/_disabled-cookie-list_access_log
T-2/var/log/httpd/_disabled-cookie-list_backend_log
T-2/var/log/httpd/_disabled-cookie-list_error_log
T-2/var/log/httpd/_empty_access_log
T-2/var/log/httpd/_empty_backend_log
T-2/var/log/httpd/_empty_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_access_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_backend_log
T-2/var/log/httpd/_enable_cache-disable-no-cache-request_error_log
T-2/var/log/httpd/_enable_cache-disable-via-header_access_log
T-2/var/log/httpd/_enable_cache-disable-via-header_backend_log
T-2/var/log/httpd/_enable_cache-disable-via-header_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_enable_cache_access_log
T-2/var/log/httpd/_enable_cache_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_access_log
T-2/var/log/httpd/_enable_cache_custom_domain_backend_log
T-2/var/log/httpd/_enable_cache_custom_domain_error_log
T-2/var/log/httpd/_enable_cache_error_log
T-2/var/log/httpd/_enable_cache_server_alias_access_log
T-2/var/log/httpd/_enable_cache_server_alias_backend_log
T-2/var/log/httpd/_enable_cache_server_alias_error_log
T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_backend_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
T-2/var/log/httpd/_monitor-ipv6-test_backend_log
T-2/var/log/httpd/_monitor-ipv6-test_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_backend_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend_error_log
T-2/var/log/httpd/_server-alias-duplicated_access_log
T-2/var/log/httpd/_server-alias-duplicated_backend_log
T-2/var/log/httpd/_server-alias-duplicated_error_log
T-2/var/log/httpd/_server-alias-empty_access_log
T-2/var/log/httpd/_server-alias-empty_backend_log
T-2/var/log/httpd/_server-alias-empty_error_log
T-2/var/log/httpd/_server-alias-wildcard_access_log
T-2/var/log/httpd/_server-alias-wildcard_backend_log
T-2/var/log/httpd/_server-alias-wildcard_error_log
T-2/var/log/httpd/_server-alias_access_log
T-2/var/log/httpd/_server-alias_backend_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_access_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_backend_log
T-2/var/log/httpd/_server-alias_custom_domain-duplicated_error_log
T-2/var/log/httpd/_server-alias_error_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_access_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_backend_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_error_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_access_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_backend_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_error_log
T-2/var/log/httpd/_ssl_ca_crt_only_access_log
T-2/var/log/httpd/_ssl_ca_crt_only_backend_log
T-2/var/log/httpd/_ssl_ca_crt_only_error_log
T-2/var/log/httpd/_type-notebook_access_log
T-2/var/log/httpd/_type-notebook_backend_log
T-2/var/log/httpd/_type-notebook_error_log
T-2/var/log/httpd/_type-redirect-custom_domain_access_log
T-2/var/log/httpd/_type-redirect-custom_domain_error_log
T-2/var/log/httpd/_type-redirect_access_log
T-2/var/log/httpd/_type-redirect_error_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_backend_log
T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_backend_log
T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_backend_log
T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
T-2/var/log/httpd/_type-websocket_access_log
T-2/var/log/httpd/_type-websocket_backend_log
T-2/var/log/httpd/_type-websocket_error_log
T-2/var/log/httpd/_type-zope-default-path_access_log
T-2/var/log/httpd/_type-zope-default-path_backend_log
T-2/var/log/httpd/_type-zope-default-path_error_log
T-2/var/log/httpd/_type-zope-path_access_log
T-2/var/log/httpd/_type-zope-path_backend_log
T-2/var/log/httpd/_type-zope-path_error_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_backend_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend-https-only_error_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_access_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_backend_log
T-2/var/log/httpd/_type-zope-prefer-gzip-encoding-to-backend_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_backend_log
T-2/var/log/httpd/_type-zope-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_access_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_backend_log
T-2/var/log/httpd/_type-zope-virtualhostroot-http-port_error_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_access_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_backend_log
T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
T-2/var/log/httpd/_type-zope_access_log
T-2/var/log/httpd/_type-zope_backend_log
T-2/var/log/httpd/_type-zope_error_log
T-2/var/log/httpd/_url_https-url_access_log
T-2/var/log/httpd/_url_https-url_backend_log
T-2/var/log/httpd/_url_https-url_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003411 13712426147 0045153 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-_monitor-ipv4-test-ipv4-packet-list-test.py
T-2/etc/plugin/check-_monitor-ipv6-test-ipv6-packet-list-test.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0044461 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlaveGlobalDisableHttp2.test_master_partition_state-CADDY.txt 0000664 0000000 0000000 00000001302 13712426147 0046404 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data SetEnvIf Origin "^http(s)?://(.+\.)?(monitor\.app\.officejs\.com)$" ORIGIN_DOMAIN=$0
Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Credentials "true" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" env=ORIGIN_DOMAIN
Header always set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" env=ORIGIN_DOMAIN
test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0045245 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0051565 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000005205 13712426147 0050245 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_error_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_access_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_backend_log
T-2/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_error_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_access_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_backend_log
T-2/var/log/httpd/_ssl_ca_crt_does_not_match_error_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_access_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_backend_log
T-2/var/log/httpd/_ssl_ca_crt_garbage_error_log
T-2/var/log/httpd/_ssl_from_master_access_log
T-2/var/log/httpd/_ssl_from_master_backend_log
T-2/var/log/httpd/_ssl_from_master_error_log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_access_log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_backend_log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_error_log
T-2/var/log/httpd/_ssl_from_slave_access_log
T-2/var/log/httpd/_ssl_from_slave_backend_log
T-2/var/log/httpd/_ssl_from_slave_error_log
T-2/var/log/httpd/_ssl_from_slave_kedifa_overrides_access_log
T-2/var/log/httpd/_ssl_from_slave_kedifa_overrides_backend_log
T-2/var/log/httpd/_ssl_from_slave_kedifa_overrides_error_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_access_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_backend_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_error_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_kedifa_overrides_access_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_kedifa_overrides_backend_log
T-2/var/log/httpd/_type-notebook-ssl_from_master_kedifa_overrides_error_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_access_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_backend_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_error_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_access_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_backend_log
T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0050762 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0050265 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0051051 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_etc_cron_d-CADDY.txt0000664 0000000 0000000 00000000644 13712426147 0054441 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001377 13712426147 0053127 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_access_log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_backend_log
T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0053636 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0053141 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0053725 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_etc_cron_d-CADDY.txt 0000664 0000000 0000000 00000000644 13712426147 0052730 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt 0000664 0000000 0000000 00000001223 13712426147 0051404 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_ssl_from_master_access_log
T-2/var/log/httpd/_ssl_from_master_backend_log
T-2/var/log/httpd/_ssl_from_master_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/trafficserver/manager.log
T-2/var/log/trafficserver/traffic.out
test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt 0000664 0000000 0000000 00000003207 13712426147 0052125 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_run-CADDY.txt 0000664 0000000 0000000 00000000621 13712426147 0051430 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt 0000664 0000000 0000000 00000003304 13712426147 0052214 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caddy-frontend/test/test_data T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase/ 0000775 0000000 0000000 00000000000 13712426147 0024617 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase/buildout.hash.cfg 0000664 0000000 0000000 00000001662 13712426147 0030056 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-caucased]
filename = instance-caucased.cfg.jinja2
md5sum = 2277c891a71534e00487468f5048d196
[instance]
filename = instance.cfg.jinja2
md5sum = d40bed5ccc457ff7dc99b618bf29b189
instance-caucase-input-schema.json 0000664 0000000 0000000 00000002446 13712426147 0033242 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"extends": "./schema-definitions.json#",
"title": "Input Parameters",
"properties": {
"base-port": {
"title": "Base TCP port",
"description": "If 80, caucase will also listen on 443. Otherwise, caucase will listen on port and port + 1.",
"type": "integer",
"default": 8009
},
"external-url": {
"$comment": "(deprecated)"
},
"service-auto-approve-amount": {
"title": "Number of service certificate requests to automatically approve",
"description": "Once that number has been reached, a user must validate further requests. Renewals do not count toward this number. Cannot be changed once set.",
"type": "integer",
"default": 0
},
"user-auto-approve-amount": {
"title": "Number of user certificate requests to automatically approve",
"description": "Once that number has been reached, a user must validate further requests. Renewals do not count toward this number. Cannot be changed once set.",
"type": "integer",
"default": 1
},
"key-length": {
"title": "Key length",
"description": "Size, in bits, of the SSL key generated to authenticate users.",
"default": 2048,
"type": "integer"
}
}
}
instance-caucase-output-schema.json 0000664 0000000 0000000 00000000361 13712426147 0033435 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Caucase instantiation",
"properties": {
"url": {
"description": "Caucase URL",
"type": "string"
}
},
"type": "object"
}
instance-caucased.cfg.jinja2 0000664 0000000 0000000 00000001550 13712426147 0031750 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase {% import "caucase" as caucase with context %}
{% set netloc = '[' ~ (ipv6_set | list)[0] ~ ']:' ~ slapparameter_dict.get('base-port', 8009) -%}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
service-on-watch = ${:etc}/service
srv = ${buildout:directory}/srv
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=bin_directory,
caucased_path='${directory:service-on-watch}/caucased',
data_dir='${directory:srv}/caucased',
netloc=netloc,
service_auto_approve_count=slapparameter_dict.get('service-auto-approve-amount', 0),
user_auto_approve_count=slapparameter_dict.get('user-auto-approve-amount', 1),
key_len=slapparameter_dict.get('key-length', 2048),
) }}
[publish]
recipe = slapos.cookbook:publish.serialised
url = {{ dumps('http://' ~ netloc) }}
[buildout]
parts =
publish
caucased
caucased-promise
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase/instance.cfg.jinja2 0000664 0000000 0000000 00000002116 13712426147 0030260 0 ustar 00root root 0000000 0000000 [buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[context]
bin-directory = {{ dumps(bin_directory) }}
caucase-jinja2-library = {{ dumps(caucase_jinja2_library) }}
instance-caucased = {{ dumps(instance_caucased) }}
[caucased]
recipe = slapos.recipe.template:jinja2
template = ${context:instance-caucased}
rendered = ${buildout:parts-directory}/instance-caucased.cfg
context =
key ipv6_set slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration
key bin_directory context:bin-directory
import-list =
file caucase context:caucase-jinja2-library
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = caucased:rendered
# XXX: When will this name finally go away ?
RootSoftwareInstance = ${:default}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase/software.cfg 0000664 0000000 0000000 00000001425 13712426147 0027134 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../stack/caucase/buildout.cfg
parts +=
instance
caucase-eggs
[instance-caucased]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
# XXX: following mode should be the default
mode = 644
[instance]
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key caucase_jinja2_library caucase-jinja2-library:target
key instance_caucased instance-caucased:target
[versions]
slapos.recipe.template = 3.0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/caucase/software.cfg.json 0000664 0000000 0000000 00000000550 13712426147 0030102 0 ustar 00root root 0000000 0000000 {
"name": "Caucase",
"description": "Caucase certificate authority",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"description": "Single caucase instance",
"request": "instance-caucase-input-schema.json",
"response": "instance-caucase-output-schema.json",
"index": 0
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me/ 0000775 0000000 0000000 00000000000 13712426147 0024356 5 ustar 00root root 0000000 0000000 instance-cdn-me-input-schema.json 0000664 0000000 0000000 00000007204 13712426147 0032535 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"port": {
"title": "Backend port",
"description": "Port where the service is running on the backend",
"default": 80,
"type": "integer"
},
"frontend-custom_domain": {
"title": "Custom Domain",
"description": "Custom Domain to use for the website",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"frontend-server-alias": {
"title": "Server Alias",
"description": "Server Alias List separated by space",
"type": "string",
"default": ""
},
"frontend-type": {
"title": "Backend Type",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster",
"type": "string",
"default": "",
"enum": [
"",
"zope",
"redirect"
]
},
"frontend-path": {
"title": "Backend Path",
"description": "Path to proxy to in the backend",
"type": "string",
"default": ""
},
"frontend-default-path": {
"title": "Default Path",
"description": "Provide default path to redirect user to",
"type": "string",
"default": ""
},
"frontend-ssl_crt": {
"title": "SSL Certificate",
"description": "SSL Certificate",
"type": "string",
"textarea": true,
"default": ""
},
"frontend-ssl_key": {
"title": "SSL Key",
"description": "SSL Key",
"type": "string",
"textarea": true,
"default": ""
},
"frontend-ssl_ca_crt": {
"title": "SSL Certificate Authority's Certificate",
"description": "SSL Key",
"type": "string",
"textarea": true,
"default": ""
},
"frontend-https-only": {
"title": "HTTPS Only",
"description": "If set to true, http request are redirect to https",
"type": "boolean",
"default": false
},
"frontend-ssl-proxy-verify": {
"title": "Verify Backend Certificates",
"description": "If set to true, Backend Certificates are checked",
"type": "boolean",
"default": false
},
"frontend-ssl_proxy_ca_crt": {
"title": "SSL Backend Authority's Certificate",
"description": "SSL Certificate Authority of the backen (to be used with ssl-proxy-verify)",
"type": "string",
"textarea": true,
"default": ""
},
"frontend-enable_cache": {
"title": "Enable Cache",
"description": "If set to true, the cache is used",
"type": "boolean",
"default": false
},
"frontend-disable-no-cache-request": {
"title": "Disable 'no-cache' requests",
"description": "If set to true, no-cache control headers will be disabled",
"type": "boolean",
"default": false
},
"frontend-disable-via-header": {
"title": "Disable 'Via' headers from cache",
"description": "If set to true, via headers will be disabled",
"type": "boolean",
"default": false
},
"frontend-prefer-gzip-encoding-to-backend": {
"title": "Prefer gzip Encoding for Backend",
"description": "If set to true, if a request is made with accept encoding 'gzip', only that one will be transferred to the backend",
"type": "boolean",
"default": false
},
"frontend-disabled-cookie-list": {
"title": "Disabled Cookies",
"description": "List of Cookies separated by space that will not be sent to the backend",
"type": "string",
"default": ""
}
}
}
instance-cdn-me-simplified-input-schema.json 0000664 0000000 0000000 00000003342 13712426147 0034657 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"port": {
"title": "Backend port",
"description": "Port where the service is running on the backend",
"default": 80,
"type": "integer"
},
"frontend-custom_domain": {
"title": "Custom Domain",
"description": "Custom Domain to use for the website",
"type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
},
"frontend-server-alias": {
"title": "Server Alias",
"description": "Server Alias List separated by space",
"type": "string",
"default": ""
},
"frontend-type": {
"title": "Backend Type",
"description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster",
"type": "string",
"default": "",
"enum": [
"",
"zope",
"redirect"
]
},
"frontend-path": {
"title": "Backend Path",
"description": "Path to proxy to in the backend",
"type": "string",
"default": ""
},
"frontend-default-path": {
"title": "Default Path",
"description": "Provide default path to redirect user to",
"type": "string",
"default": ""
},
"frontend-https-only": {
"title": "HTTPS Only",
"description": "If set to true, http request are redirect to https",
"type": "boolean",
"default": false
},
"frontend-enable_cache": {
"title": "Enable Cache",
"description": "If set to true, the cache is used",
"type": "boolean",
"default": false
}
}
}
instance-cdn-request.cfg.jinja2 0000664 0000000 0000000 00000013153 13712426147 0032173 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me [buildout]
parts =
request-monitor-test-distributor-slave
publish-connection-informations
connection-parameter-http-checker
connection-parameter-checker
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
promise = $${:home}/etc/promise
etc = $${:home}/etc/
var = $${:home}/var/
srv = $${:home}/srv/
bin = $${:home}/bin/
tmp = $${:home}/tmp/
[request-frontend-token-slave]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = WebSite Frontend
software-url = product.frontend
software-type = custom-personal
slave = true
{% if slapparameter_dict.get('port', '80') != '443' %}
config-url = http://[$${request-re6stnet-token-slave:connection-ipv6}1]:{{ slapparameter_dict.get('port', '80') }}
{% endif -%}
{% if slapparameter_dict.get('port', '80') == '443' %}
config-url = https://[$${request-re6stnet-token-slave:connection-ipv6}1]:{{ slapparameter_dict.get('port', '443') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-custom_domain', '') %}
config-custom_domain = {{ slapparameter_dict.get('frontend-custom_domain', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-server-alias', '') %}
config-server-alias = {{ slapparameter_dict.get('frontend-server-alias', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-type', '') %}
config-type = {{ slapparameter_dict.get('frontend-type', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-path', '') %}
config-path = {{ slapparameter_dict.get('frontend-path', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-default-path', '') %}
config-default-path = {{ slapparameter_dict.get('frontend-default-path', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-ssl_crt', '') %}
config-ssl_crt = {{ slapparameter_dict.get('frontend-ssl_crt', '').split('\n') | join('\n ') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-ssl_key', '') %}
config-ssl_key = {{ slapparameter_dict.get('frontend-ssl_key', '').split('\n') | join('\n ') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-ssl_ca_crt', '') %}
config-ssl_ca_crt = {{ slapparameter_dict.get('frontend-ssl_ca_crt', '').split('\n') | join('\n ') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-https-only', '') %}
config-https-only = {{ slapparameter_dict.get('frontend-https-only', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-ssl-proxy-verify', '') %}
config-ssl-proxy-verify = {{ slapparameter_dict.get('frontend-ssl-proxy-verify', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-ssl_proxy_ca_crt', '') %}
config-ssl_proxy_ca_crt = {{ slapparameter_dict.get('frontend-ssl_proxy_ca_crt', '').split('\n') | join('\n ') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-enable_cache', '') %}
config-enable_cache = {{ slapparameter_dict.get('frontend-enable_cache', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-disable-no-cache-request', '') %}
config-disable-no-cache-request = {{ slapparameter_dict.get('frontend-disable-no-cache-request', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-disable-via-header', '') %}
config-disable-via-header = {{ slapparameter_dict.get('frontend-disable-via-header', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-prefer-gzip-encoding-to-backend', '') %}
config-prefer-gzip-encoding-to-backend = {{ slapparameter_dict.get('frontend-prefer-gzip-encoding-to-backend', '') }}
{% endif -%}
{% if slapparameter_dict.get('frontend-disabled-cookie-list', '') %}
config-disabled-cookie-list = {{ slapparameter_dict.get('frontend-disabled-cookie-list', '') }}
{% endif -%}
return = site_url domain
[request-re6stnet-token-slave]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Re6st token Frontend
# XXX We have hardcoded SR URL here.
software-url = product.re6st
slave = true
return = token info_1 ipv6 ipv4
[request-monitor-test-distributor-slave]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Monitor Test
# XXX We have hardcoded SR URL here.
software-url = product.monitor
software-type = distributor
slave = true
config-ping6_ip_list = $${request-re6stnet-token-slave:connection-ipv6}
config-ping_ip_list = $${request-re6stnet-token-slave:connection-ipv4}
return = site_url
[monitor-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Monitor Frontend
# XXX We have hardcoded SR URL here.
software-url = product.frontend
software-type = custom-personal
slave = true
config-url = $${request-monitor-test-distributor-slave:connection-site_url}
return = site_url domain
[publish-connection-informations]
recipe = slapos.cookbook:publish
url-https = https://$${request-frontend-token-slave:connection-domain}
url = http://$${request-frontend-token-slave:connection-domain}
token = $${request-re6stnet-token-slave:connection-token}
ipv6 = $${request-re6stnet-token-slave:connection-ipv6}
ipv4 = $${request-re6stnet-token-slave:connection-ipv4}
monitor_v6_url = $${request-monitor-test-distributor-slave:connection-site_url}
monitor_url = https://$${monitor-frontend:connection-domain}
info_1 = $${request-re6stnet-token-slave:connection-info_1}
[connection-parameter-checker]
recipe = slapos.cookbook:check_parameter
path = $${directory:promise}/check_re6stnet_ipv6
value = $${request-re6stnet-token-slave:connection-ipv6}
expected-not-value = ::
expected-value =
[connection-parameter-http-checker]
recipe = slapos.cookbook:check_parameter
path = $${directory:promise}/check_re6stnet_http
value = $${publish-connection-informations:url}
expected-not-value = http://
expected-value =
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me/instance.cfg 0000664 0000000 0000000 00000001656 13712426147 0026653 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-base-runner:rendered}
[instance-base-runner]
recipe = slapos.recipe.template:jinja2
template = ${template-cdn-request:output}
rendered = $${buildout:directory}/template-cdn.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
mode = 0644
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me/software.cfg 0000664 0000000 0000000 00000001642 13712426147 0026674 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
# stacks are listed from most generic to most specific,
# to avoid versioning issues
parts =
template
eggs
template-cdn-request
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = a91fe7c80720d57c2acbf606a6a0d84d
mode = 0644
[template-cdn-request]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-cdn-request.cfg.jinja2
output = ${buildout:directory}/template-cdn-request.cfg.jinja2
md5sum = a5ee428ec915a9e638b46ce3f4dd42f3
mode = 0644
[eggs]
recipe = zc.recipe.egg
eggs =
collective.recipe.environment
cns.recipe.symlink
erp5.util
lock-file
plone.recipe.command
slapos.recipe.build
${slapos-cookbook:eggs}
[versions]
cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
plone.recipe.command = 1.1
slapos.recipe.template = 4.4
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cdn-me/software.cfg.json 0000664 0000000 0000000 00000001133 13712426147 0027637 0 ustar 00root root 0000000 0000000 {
"name": "CDN ME",
"description": "CDN ME",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Re6st registry",
"request": "instance-cdn-me-simplified-input-schema.json",
"response": "instance-cdn-me-output-schema.json",
"index": 0
},
"default-advanced": {
"title": "Default (Advanced Form)",
"software-type": "default",
"description": "Re6st registry",
"request": "instance-cdn-me-input-schema.json",
"response": "instance-cdn-me-output-schema.json",
"index": 1
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/ 0000775 0000000 0000000 00000000000 13712426147 0025036 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/buildout.hash.cfg 0000664 0000000 0000000 00000001611 13712426147 0030267 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-cloudooo]
filename = instance.cfg.in
md5sum = 6e4431cf4b0a0d034402604b1e2844c0
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = e49ee3e309a19f9a3a9590789690c611
instance-cloudooo-input-schema.json 0000664 0000000 0000000 00000001532 13712426147 0033673 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo {
"$schema": "http://json-schema.org/draft-04/schema#",
"extends": "./schema-definitions.json#",
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward",
"default": 23000
}
]
},
"backend-count": {
"description": "Number of backend cloudooo instances",
"default": 1,
"type": "integer"
},
"timeout": {
"description": "Configure apache with this timeout",
"type": "integer"
},
"mimetype-entry-addition": {
"description": "The list of entry to add to the cloudooo mimetype registry. Each entry should on one line which format is: \" \"",
"type": "string"
}
}
}
instance-cloudooo-output-schema.json 0000664 0000000 0000000 00000000414 13712426147 0034072 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Cloudooo instanciation",
"properties": {
"url": {
"description": "Conversion service access information",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/instance-cloudooo.cfg.in 0000664 0000000 0000000 00000024741 13712426147 0031561 0 ustar 00root root 0000000 0000000 {% set ipv4 = (ipv4_set | list)[0] -%}
{% if ipv6_set %}{% set ipv6 = (ipv6_set | list)[0] %}{% endif -%}
{% set instance_parameter_dict = parameter_dict['instance-parameter-dict'] -%}
{% macro assert(x) %}{{ ("",)[not x] }}{% endmacro -%}
{% set publish_url_name = instance_parameter_dict.get('publish-url-name') or 'url' -%}
{% set next_port = instance_parameter_dict['port'] | int -%}
{% if instance_parameter_dict.get('port-parameter-name') -%}
{% set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%}
{% endif -%}
{% do assert(next_port > 0) -%}
{% set next_port = itertools.count(next_port).next -%}
{% set backend_count = instance_parameter_dict['backend-count'] | int -%}
{% if instance_parameter_dict.get('backend-count-parameter-name') -%}
{% set backend_count = slapparameter_dict.get(instance_parameter_dict['backend-count-parameter-name'], backend_count) | int -%}
{% endif -%}
{% do assert(backend_count > 0) -%}
{% set timeout = instance_parameter_dict['timeout'] | int -%}
{% if instance_parameter_dict.get('timeout-parameter-name') -%}
{% set timeout = slapparameter_dict.get(instance_parameter_dict['timeout-parameter-name'], timeout) | int -%}
{% endif -%}
{% do assert(timeout > 0) -%}
{% set mimetype_entry_addition = instance_parameter_dict.get('mimetype-entry-addition', '') -%}
{% if instance_parameter_dict.get('mimetype-entry-addition-parameter-name') -%}
{% set mimetype_entry_addition = mimetype_entry_addition ~ "\n" ~ slapparameter_dict.get(instance_parameter_dict['mimetype-entry-addition-parameter-name'], '') -%}
{% endif -%}
{% set apache_port = next_port() -%}
{% set haproxy_port = next_port() -%}
{% set apache_ip_list = [ipv4] -%}
{% if ipv6_set -%}
{% do apache_ip_list.append('[' ~ ipv6 ~ ']') -%}
{% endif -%}
{% set apache_dict = {} -%}
{% do apache_dict.__setitem__(publish_url_name, (apache_port, "https", 'http://' ~ ipv4 ~ ':' ~ haproxy_port, False)) -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
{% set section_list = [] -%}
{% set cloudooo_section_list = [] -%}
{% macro cloudooo(name) %}{% do cloudooo_section_list.append(name) %}{{ name }}{% endmacro -%}
[simplefile]
recipe = slapos.recipe.template:jinja2
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section_name }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{ content_section_name }}:content
mode = {{ mode }}
{%- endmacro %}
[buildout]
extends =
{{ parameter_dict['template-logrotate-base'] }}
{{ parameter_dict['template-monitor'] }}
parts =
monitor-base
publish
apache
apache-conf
apache-promise
apache-logrotate
cloudooo-test-runner
haproxy
xvfb-instance
wkhtmltopdf-on-xvfb
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[apache]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/apache
command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
[apache-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-apache-conf'] }}
rendered = ${directory:apache-conf}/apache.conf
context = section parameter_dict apache-conf-parameter-dict
[apache-conf-parameter-dict]
backend-list = {{ dumps(apache_dict.values()) }}
ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid
error-log = ${directory:log}/apache-error.log
access-log = ${directory:log}/apache-access.log
# Apache 2.4's default value (60 seconds) can be a bit too short
timeout = {{ timeout }}
# Basic SSL server configuration
cert = ${apache-ssl:cert}
key = ${apache-ssl:key}
cipher =
ssl-session-cache = ${directory:log}/apache-ssl-session-cache
[apache-promise]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_url_available
name = apache.py
config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
# XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
# is that we don't have a "503 Service Unavailable" from apache or haproxy.
config-http_code = 400
[apache-conf-ssl]
cert = ${directory:apache-conf}/apache.crt
key = ${directory:apache-conf}/apache.pem
ca-cert = ${directory:apache-conf}/ca.crt
crl = ${directory:apache-conf}/crl.pem
[apache-ssl]
recipe = plone.recipe.command
command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
key = ${apache-conf-ssl:key}
cert = ${apache-conf-ssl:cert}
[apache-logrotate]
< = logrotate-entry-base
name = apache
log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
[publish]
recipe = slapos.cookbook:publish.serialised
{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
{% endfor -%}
[fontconfig-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-fonts-conf'] }}
rendered = ${directory:etc}/fonts.conf
context =
key cachedir directory:fontconfig-cache
key fonts :fonts
key includes :includes
fonts =
{% for font in parameter_dict['fonts'].splitlines() %}
{{ font }}
{% endfor%}
${directory:font}
includes =
{% for include in parameter_dict['fontconfig-includes'].splitlines() %}
{{ include }}
{% endfor%}
[cloudooo-base]
recipe = slapos.cookbook:generic.cloudooo
ip = {{ ipv4 }}
environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory}
mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }}
{% endfor -%}
# Binary information
# cloudooo specific configuration
ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%}
{% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}]
< = cloudooo-base
port = {{ next_port() }}
openoffice-port = {{ next_port() }}
configuration-file = ${directory:etc}/{{ name }}.cfg
data-directory = ${directory:srv}/{{ name }}
wrapper = ${directory:services}/{{ name }}
{% endfor -%}
[haproxy]
recipe = slapos.cookbook:haproxy
name = cloudooo
conf-path = ${directory:etc}/haproxy.cfg
socket-path = ${directory:run}/haproxy.sock
ip = {{ ipv4 }}
port = {{ haproxy_port }}
maxconn = 1
wrapper-path = ${directory:services}/haproxy
binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy
ctl-path = ${directory:bin}/haproxy-ctl
backend-list =
{%- for section_name in cloudooo_section_list %}
{{ "${" ~ section_name ~ ":ip}:${" ~ section_name ~ ":port}" }}
{%- endfor %}
[cloudooo-test-runner]
recipe = slapos.cookbook:cloudooo.test
prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
[binary-link]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary =
{{ parameter_dict['coreutils'] }}/bin/basename
{{ parameter_dict['coreutils'] }}/bin/cat
{{ parameter_dict['coreutils'] }}/bin/cp
{{ parameter_dict['coreutils'] }}/bin/ls
{{ parameter_dict['coreutils'] }}/bin/tr
{{ parameter_dict['coreutils'] }}/bin/uname
{{ parameter_dict['coreutils'] }}/bin/dirname
# wrapper recipe needs the head command
{{ parameter_dict['coreutils'] }}/bin/head
{{ parameter_dict['imagemagick'] }}/bin/convert
{{ parameter_dict['imagemagick'] }}/bin/identify
{{ parameter_dict['poppler'] }}/bin/pdfinfo
{{ parameter_dict['poppler'] }}/bin/pdftotext
{{ parameter_dict['poppler'] }}/bin/pdftohtml
{{ parameter_dict['onlyoffice-core'] }}/bin/x2t
# rest of parts are candidates for some generic stuff
[directory]
recipe = slapos.cookbook:mkdirectory
apache-conf = ${:etc}/apache
bin = ${buildout:directory}/bin
ca-dir = ${buildout:directory}/srv/ssl
certs = ${:ca-dir}/certs
crl = ${:ca-dir}/crl
etc = ${buildout:directory}/etc
font = ${:srv}/font
fontconfig-cache = ${buildout:directory}/.fontconfig
log = ${:var}/log
newcerts = ${:ca-dir}/newcerts
private = ${:ca-dir}/private
requests = ${:ca-dir}/requests
run = ${:var}/run
services = ${:etc}/run
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
framebuffer = ${:srv}/framebuffer
[xvfb-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/${:_buildout_section_name_}
command-line =
{{ parameter_dict["xserver"] }}/bin/Xvfb
${:display}
-screen 0 1024x768x24
-fbdir ${directory:framebuffer}
environment =
XORG_LOCK_DIR=${:lock-dir}
display = :0
lock-dir = ${directory:run}
[wkhtmltopdf-on-xvfb]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/wkhtmltopdf
environment =
DISPLAY=${xvfb-instance:display}
XORG_LOCK_DIR=${xvfb-instance:lock-dir}
command-line = {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf --use-xserver
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/instance.cfg.in 0000664 0000000 0000000 00000003404 13712426147 0027731 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extra-context =
context =
key ipv6_set slap-parameters:ipv6
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slapparameter_dict slap-parameters:configuration
${:extra-context}
[cloudooo-dynamic-template-parameter-dict]
{% for key, value in dynamic_template_cloudooo_instance_parameter_dict.items() -%}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
instance-parameter-dict = {{ dumps(cloudooo_parameter_dict) }}
[cloudooo-dynamic-template]
< = jinja2-template-base
template = {{ template_cloudooo_instance }}
filename = instance-cloudooo.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict cloudooo-dynamic-template-parameter-dict
key ipv4_set slap-parameters:ipv4
import itertools itertools
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${cloudooo-dynamic-template:rendered}
[slap-connection]
# part to migrate to new - separated words
computer-id = ${slap_connection:computer_id}
partition-id = ${slap_connection:partition_id}
server-url = ${slap_connection:server_url}
software-release-url = ${slap_connection:software_release_url}
key-file = ${slap_connection:key_file}
cert-file = ${slap_connection:cert_file}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/software-common.cfg 0000664 0000000 0000000 00000006163 13712426147 0030645 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../stack/cloudooo.cfg
../../stack/logrotate/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/defaults.cfg
parts =
${cloudooo-buildout:parts}
[cloudooo-buildout]
parts =
${stack-cloudooo-buildout:parts}
# Local development
cloudooo-develop
slapos-cookbook
[slap-parameters]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[template-cloudooo]
< = template-cloudooo-base
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 640
[template-cloudooo-base]
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key template_cloudooo_instance template-cloudooo-instance:target
section dynamic_template_cloudooo_instance_parameter_dict dynamic-template-cloudooo-instance-parameter-dict
section cloudooo_parameter_dict cloudooo-software-parameter-dict
[dynamic-template-cloudooo-instance-parameter-dict]
apache = ${apache:location}
buildout-bin-directory = ${buildout:bin-directory}
cairo = ${cairo:location}
coreutils = ${coreutils:location}
cups = ${cups:location}
dash = ${dash:location}
dbus = ${dbus:location}
dbus-glib = ${dbus-glib:location}
file = ${file:location}
fontconfig = ${fontconfig:location}
template-fonts-conf = ${template-fonts-conf:output}
fonts =
${android-fonts:location}
${ipa-fonts:location}
${ipaex-fonts:location}
${liberation-fonts:location}
${ocrb-fonts:location}
${dejavu-fonts:location}
${libreoffice-bin:location}/share/fonts/
fontconfig-includes =
${fontconfig:location}/etc/fonts/conf.d
freetype = ${freetype:location}
gcc = ${gcc:prefix}
glib = ${glib:location}
glu = ${glu:location}
haproxy = ${haproxy:location}
imagemagick = ${imagemagick:location}
libICE = ${libICE:location}
libSM = ${libSM:location}
libX11 = ${libX11:location}
libXau = ${libXau:location}
libXdmcp = ${libXdmcp:location}
libXext = ${libXext:location}
libXrender = ${libXrender:location}
libexpat = ${libexpat:location}
libffi = ${libffi:location}
libpng12 = ${libpng12:location}
libreoffice-bin = ${libreoffice-bin:location}
libxcb = ${libxcb:location}
mesa = ${mesa:location}
openssl = ${openssl:location}
onlyoffice-core = ${onlyoffice-core:location}
poppler = ${poppler:location}
pixman = ${pixman:location}
wkhtmltopdf = ${wkhtmltopdf:location}
xdamage = ${xdamage:location}
xfixes = ${xfixes:location}
xserver = ${xserver:location}
zlib = ${zlib:location}
template-apache-conf = ${template-apache-backend-conf:target}
template-logrotate-base = ${template-logrotate-base:rendered}
template-monitor = ${monitor2-template:rendered}
[template-cloudooo-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 640
[versions]
# Required by:
# cloudooo==1.2.6.dev0
argparse = 1.4.0
# Required by:
# cloudooo==1.2.6.dev0
pyPdf = 1.13
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/software.cfg 0000664 0000000 0000000 00000001004 13712426147 0027344 0 ustar 00root root 0000000 0000000 [buildout]
extends =
software-common.cfg
parts =
${cloudooo-buildout:parts}
template-cloudooo
[cloudooo-software-parameter-dict]
publish-url-name = cloudooo
port-parameter-name = tcpv4-port
port = 8000
backend-count-parameter-name = backend-count
backend-count = 1
timeout-parameter-name = timeout
# timeout in seconds
timeout = 600
ssl-dict-parameter-name = ssl
#ssl-dict =
mimetype-entry-addition-parameter-name = mimetype-entry-addition
#mimetype-entry-addition =
# text/html application/pdf wkhtmltopdf
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/software.cfg.json 0000664 0000000 0000000 00000000513 13712426147 0030320 0 ustar 00root root 0000000 0000000 {
"name": "cloudooo",
"description": "Clusterised cloudooo",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Cloudooo",
"request": "instance-cloudooo-input-schema.json",
"response": "instance-cloudooo-output-schema.json",
"index": 0
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/test/ 0000775 0000000 0000000 00000000000 13712426147 0026015 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/test/README.md 0000664 0000000 0000000 00000000044 13712426147 0027272 0 ustar 00root root 0000000 0000000 Tests for Cloudooo software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/test/setup.py 0000664 0000000 0000000 00000003726 13712426147 0027537 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.cloudooo'
with open("README.md") as f:
long_description = f.read()
setup(name=name,
version=version,
description="Test for SlapOS' cloudooo",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'six',
'PyPDF2',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cloudooo/test/test.py 0000664 0000000 0000000 00000020152 13712426147 0027346 0 ustar 00root root 0000000 0000000 ##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import six.moves.xmlrpc_client as xmlrpclib
import ssl
import base64
import io
import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30
def normalizeFontName(font_name):
if '+' in font_name:
return font_name.split('+')[1]
if font_name.startswith('/'):
return font_name[1:]
def getReferencedFonts(pdf_file_reader):
"""Return fonts referenced in this pdf
"""
fonts = set()
def collectFonts(obj):
"""Recursively visit PDF objects and collect referenced fonts in `fonts`
"""
if hasattr(obj, 'keys'):
if '/BaseFont' in obj:
fonts.add(obj['/BaseFont'])
for k in obj.keys():
collectFonts(obj[k])
for page in pdf_file_reader.pages:
collectFonts(page.getObject()['/Resources'])
return {normalizeFontName(font) for font in fonts}
class HTMLtoPDFConversionFontTestMixin:
"""Mix-In class to test how fonts are selected during
HTML to PDF conversions.
This needs to be mixed with a test case defining:
* pdf_producer : the name of /Producer in PDF metadata
* expected_font_mapping : a mapping of resulting font name in pdf,
keyed by font-family in the input html
* _convert_html_to_pdf: a method to to convert html to pdf
"""
def _convert_html_to_pdf(self, src_html):
# type: (str) -> bytes
"""Convert the HTML source to pdf bytes.
"""
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self):
actual_font_mapping_mapping = {}
for font in self.expected_font_mapping:
src_html = '''
the quick brown fox jumps over the lazy dog.
THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.
'''.format(**locals())
pdf_data = self._convert_html_to_pdf(src_html)
pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
self.assertEqual(
self.pdf_producer,
pdf_reader.getDocumentInfo()['/Producer'])
fonts_in_pdf = getReferencedFonts(pdf_reader)
if len(fonts_in_pdf) == 1:
actual_font_mapping_mapping[font] = fonts_in_pdf.pop()
else:
actual_font_mapping_mapping[font] = fonts_in_pdf
self.maxDiff = None
self.assertEqual(self.expected_font_mapping, actual_font_mapping_mapping)
class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
__partition_reference__ = 'wk'
pdf_producer = 'Qt 4.8.7'
expected_font_mapping = {
'Arial': 'LiberationSans',
'Arial Black': 'LiberationSans',
'Avant Garde': 'LiberationSans',
'Bookman': 'LiberationSans',
'Carlito': 'Carlito',
'Comic Sans MS': 'LiberationSans',
'Courier New': 'LiberationSans',
'DejaVu Sans': 'DejaVuSans',
'DejaVu Sans Condensed': 'LiberationSans',
'DejaVu Sans ExtraLight': 'LiberationSans',
'DejaVu Sans Mono': 'DejaVuSansMono',
'DejaVu Serif': 'DejaVuSerif',
'DejaVu Serif Condensed': 'LiberationSans',
'Garamond': 'LiberationSans',
'Gentium Basic': 'GentiumBasic',
'Gentium Book Basic': 'GentiumBookBasic',
'Georgia': 'LiberationSans',
'Helvetica': 'LiberationSans',
'IPAex Gothic': 'LiberationSans',
'IPAex Mincho': 'LiberationSans',
'Impact': 'LiberationSans',
'Liberation Mono': 'LiberationMono',
'Liberation Sans': 'LiberationSans',
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LiberationSans',
'OpenSymbol': set(['DejaVuSans', 'OpenSymbol']),
'Palatino': 'LiberationSans',
'Roboto Black': 'LiberationSans',
'Roboto Condensed Light': 'LiberationSans',
'Roboto Condensed Regular': 'LiberationSans',
'Roboto Light': 'LiberationSans',
'Roboto Medium': 'LiberationSans',
'Roboto Thin': 'LiberationSans',
'Times New Roman': 'LiberationSans',
'Trebuchet MS': 'LiberationSans',
'Verdana': 'LiberationSans',
'ZZZdefault fonts when no match': 'LiberationSans'
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'pdf',
False,
False,
{
'encoding': 'utf-8'
},
).encode())
class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
__partition_reference__ = 'lo'
pdf_producer = 'LibreOffice 5.2'
expected_font_mapping = {
'Arial': 'LiberationSans',
'Arial Black': 'DejaVuSans',
'Avant Garde': 'DejaVuSans',
'Bookman': 'DejaVuSans',
'Carlito': 'Carlito',
'Comic Sans MS': 'DejaVuSans',
'Courier New': 'LiberationMono',
'DejaVu Sans': 'DejaVuSans',
'DejaVu Sans Condensed': 'DejaVuSansCondensed',
'DejaVu Sans ExtraLight': 'DejaVuSans',
'DejaVu Sans Mono': 'DejaVuSansMono',
'DejaVu Serif': 'DejaVuSerif',
'DejaVu Serif Condensed': 'DejaVuSerifCondensed',
'Garamond': 'DejaVuSerif',
'Gentium Basic': 'GentiumBasic',
'Gentium Book Basic': 'GentiumBookBasic',
'Georgia': 'DejaVuSerif',
'Helvetica': 'LiberationSans',
'IPAex Gothic': 'IPAexGothic',
'IPAex Mincho': 'IPAexMincho',
'Impact': 'DejaVuSans',
'Liberation Mono': 'LiberationMono',
'Liberation Sans': 'LiberationSans',
'Liberation Sans Narrow': 'LiberationSansNarrow',
'Liberation Serif': 'LiberationSerif',
'Linux LibertineG': 'LinuxLibertineG',
'OpenSymbol': 'OpenSymbol',
'Palatino': 'DejaVuSerif',
'Roboto Black': 'Roboto-Black',
'Roboto Condensed Light': 'RobotoCondensed-Light',
'Roboto Condensed Regular': 'DejaVuSans',
'Roboto Light': 'Roboto-Light',
'Roboto Medium': 'Roboto-Medium',
'Roboto Thin': 'Roboto-Thin',
'Times New Roman': 'LiberationSerif',
'Trebuchet MS': 'DejaVuSans',
'Verdana': 'DejaVuSans',
'ZZZdefault fonts when no match': 'DejaVuSans'
}
def _convert_html_to_pdf(self, src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'pdf',
).encode())
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cython-test/ 0000775 0000000 0000000 00000000000 13712426147 0025474 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cython-test/runTestSuite.in 0000664 0000000 0000000 00000007622 13712426147 0030511 0 ustar 00root root 0000000 0000000 #!{{ runTestSuite_py }}
"""
Script to run the Cython test suite using Nexedi's test node framework.
"""
import argparse, json, os, subprocess, sys, traceback
from time import gmtime, strftime, time
from unittest import TextTestResult
from erp5.util import taskdistribution, testsuite
os.environ['TEMP'] = {{ repr(tmpdir) }}
command = """. {{ cython_env_sh }}
make all test
"""
def parseTestStdOut(data):
"""
Parse output of Cython testrunner script.
"""
data = data.rsplit(b'\n' + TextTestResult.separator2.encode() + b'\n', 1)[1]
status_dict = {}
search = testsuite.TestSuite.RUN_RE.search(data)
if search:
groupdict = search.groupdict()
status_dict.update(duration=float(groupdict['seconds']),
test_count=int(groupdict['all_tests']))
search = testsuite.TestSuite.STATUS_RE.search(data)
if search:
groupdict = search.groupdict()
status_dict.update(
error_count=int(groupdict['errors'] or 0),
failure_count=int(groupdict['failures'] or 0)
+int(groupdict['unexpected_successes'] or 0),
skip_count=int(groupdict['skips'] or 0)
+int(groupdict['expected_failures'] or 0))
return status_dict
class DummyTestResult:
class DummyTestResultLine:
def stop(self, **kw):
with open(self.name + '.json', 'w') as f:
json.dump(kw, f)
done = 0
def __init__(self, test_name_list):
self.test_name_list = test_name_list
def start(self):
test_result_line = self.DummyTestResultLine()
try:
test_result_line.name = self.test_name_list[self.done]
except IndexError:
return
self.done += 1
return test_result_line
def main():
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', help='ignored', type=int)
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
args = parser.parse_args()
test_suite_title = args.test_suite_title or args.test_suite
test_name_list = 'cython',
if args.master_url:
tool = taskdistribution.TaskDistributor(portal_url = args.master_url)
test_result = tool.createTestResult(args.revision,
test_name_list,
args.test_node_title,
test_title=test_suite_title,
project_title=args.project_title)
if test_result is None:
return
else:
test_result = DummyTestResult(test_name_list)
# run NEO tests
while 1:
test_result_line = test_result.start()
if not test_result_line:
break
try:
with open(os.devnull) as stdin:
p = subprocess.Popen(command, shell=True, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd={{repr(cython_repository)}})
except Exception:
end = time()
stderr = traceback.format_exc()
status_dict = {}
sys.stderr.write(stderr)
else:
stdout, stderr = p.communicate()
end = time()
os.write(1, stdout)
os.write(2, stderr)
status_dict = parseTestStdOut(stderr)
if str is not bytes:
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
status_dict['stdout'] = stdout
# report status back to Nexedi ERP5
test_result_line.stop(
command=command,
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
stderr=stderr,
**status_dict)
if __name__ == "__main__":
main()
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/cython-test/software.cfg 0000664 0000000 0000000 00000005604 13712426147 0030014 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../component/numpy/buildout.cfg
parts =
slapos-cookbook
template
[python]
part = python3
[template]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template =
inline:[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
parts = runTestSuite
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template = ${runTestSuite.in:target}
mode = 0755
context =
key tmpdir directory:tmp
key slapparameter_dict slap-configuration:configuration
key cython_repository cython-repository:location
raw runTestSuite_py ${runTestSuite_py:bin-directory}/${runTestSuite_py:interpreter}
raw cython_env_sh ${cython_env.sh:rendered}
[cython-repository]
recipe = slapos.recipe.build:gitclone
repository = ${cython-repository:location}
git-executable = ${git:location}/bin/git
shared = true
[cython-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cython.git
git-executable = ${git:location}/bin/git
sparse-checkout = /.gitignore
[runTestSuite.in]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
md5sum = 02094e80cde9631081077fc96b401065
[runTestSuite_py]
recipe = zc.recipe.egg
eggs = erp5.util
interpreter = ${:_buildout_section_name_}
[eggs]
recipe = zc.recipe.egg
eggs =
${numpy:egg}
coverage
pycodestyle
[cython_env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template =
inline:{% set path, python = os.path.split(python) -%}
{% if 'part' in gcc -%}
{% set path = path + ':' + gcc.prefix + '/bin' -%}
{% endif -%}
export PATH={{ path }}:$PATH
export PYTHON={{ python }}
export PYTHONPATH={{ ':'.join(easy_install.working_set(eggs['eggs'].split(), [
eggs['develop-eggs-directory'],
eggs['eggs-directory'],
]).entries) }}$${PYTHONPATH:+:$PYTHONPATH}
# EmbedTest needs libintl.
export LD_RUN_PATH={{ gettext }}/lib$${LD_RUN_PATH:+:$LD_RUN_PATH}
export LIBRARY_PATH={{ gettext }}/lib$${LIBRARY_PATH:+:$LIBRARY_PATH}
{##}
context =
section eggs eggs
section gcc gcc
key gettext gettext:location
key python python:executable
import os os
import easy_install zc.buildout.easy_install
[versions]
slapos.recipe.template = 4.4
coverage = 4.5.1
numpy = 1.16.4
pycodestyle = 2.5.0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/ 0000775 0000000 0000000 00000000000 13712426147 0025352 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/buildout.hash.cfg 0000664 0000000 0000000 00000000261 13712426147 0030603 0 ustar 00root root 0000000 0000000 [template]
filename = instance.cfg
md5sum = bed788dee6daf05349c4577e7a7f1299
[instance-davstorage]
filename = instance-davstorage.cfg
md5sum = 081f97f9e575af7a4b7f9afc7c80dd4a
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/common.cfg 0000664 0000000 0000000 00000002266 13712426147 0027331 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../component/apache-php/buildout.cfg
../../component/apache/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/gzip/buildout.cfg
./buildout.hash.cfg
parts =
apache-php
application
template
lxml-python
instance-recipe-egg
[instance-recipe]
egg = slapos.cookbook
module = davstorage
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[application]
recipe = hexagonit.recipe.download
ignore-existing = true
url = http://garr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/dev-channel/4.3.4/ajaxplorer-core-4.3.4.tar.gz
md5sum = 2f2ff8bda7bbe841ef0e870c724eb74f
strip-top-level-dir = true
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[instance-davstorage]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-davstorage.cfg
mode = 0644
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/development.cfg 0000664 0000000 0000000 00000001671 13712426147 0030362 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/git/buildout.cfg
common.cfg
parts +=
slapos.cookbook-repository
slapos.toolbox-repository
check-recipe
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
branch = davstorage-ajaxplorer
git-executable = ${git:location}/bin/git
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = master
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[versions]
openssl = 1.0.1c
slapos.recipe.template = 2.4.2
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/instance-davstorage.cfg0000664 0000000 0000000 00000013653 13712426147 0032004 0 ustar 00root root 0000000 0000000 [buildout]
parts =
davstorage
publish-connection-informations
certificate-authority
ca-davstorage
cron
cron-entry-logrotate
logrotate
logrotate-entry-davstorage
request-frontend
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[publish-connection-informations]
recipe = slapos.cookbook:publish
webdav_access = https://$${request-frontend:connection-domain}
url = https://$${request-frontend-ajaxupload:connection-domain}
webdav_user = $${davstorage:user}
webdav_password = $${davstorage:password}
[davstorage]
recipe = slapos.cookbook:davstorage
user = user
password = insecure
port_webdav = 8080
port_ajax = 8070
ip = $${slap-network-information:global-ipv6}
# Path
wrapper = $${rootdirectory:bin}/davstorage
error-log = $${directory:davstorage-log}/error.log
access-log = $${directory:davstorage-log}/access.log
pid-file = $${basedirectory:run}/davstorage.pid
lock-file = $${basedirectory:run}/davstorage.lock
davdb-lock = $${basedirectory:run}/davdb.lock
key-file = $${ca-davstorage:key-file}
cert-file = $${ca-davstorage:cert-file}
conf-file = $${directory:davstorage-conf}/davstorage.conf
htdocs = $${directory:htdocs}
root = $${buildout:directory}
email-address = admin+davstorage@vifib.net
htpasswd-file = $${directory:davstorage-conf}/davstorage.htpasswd
promise = $${basedirectory:promises}/davstorage
php-ini-dir = $${directory:php-ini-dir}
tmp-dir = $${directory:tmp-php}
# Binaries
apache-binary = ${apache:location}/bin/httpd
apache-modules-dir = ${apache:location}/modules/
apache-mime-file = ${apache:location}/conf/mime.types
apache-htpasswd = ${apache:location}/bin/htpasswd
source = ${application:location}
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${basedirectory:services}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[logrotate-entry-davstorage]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = davstorage
log = $${davstorage:error-log} $${davstorage:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
[ca-davstorage]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${directory:davstorage-conf}/davstorage.key
cert-file = $${directory:davstorage-conf}/davstorage.crt
executable = $${davstorage:wrapper}
wrapper = $${basedirectory:services}/davstorage
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www/
davstorage-log = $${basedirectory:log}/davstorage/
logrotate-backup = $${basedirectory:backup}/logrotate/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
davstorage-conf = $${rootdirectory:etc}/davstorage/
ca-dir = $${rootdirectory:srv}/ssl/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
cronstamps = $${rootdirectory:etc}/cronstamps/
cronoutput = $${basedirectory:log}/cron/
php-ini-dir = $${rootdirectory:etc}/php
tmp-php = $${rootdirectory:tmp}/php
# Request frontend
[request-frontend-ajaxupload]
<= slap-connection
recipe = slapos.cookbook:request
name = Frontend Ajax
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-https-only = true
config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
return = domain
[request-frontend]
<= slap-connection
recipe = slapos.cookbook:request
name = Frontend Webdav
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-https-only = true
config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
return = domain
# XXX Vivien: promise not working for now
#[frontend-ajaxupload-promise]
#recipe = slapos.cookbook:check_url_available
#path = $${basedirectory:promises}/frontend-ajaxupload
#url = $${request-frontend-ajaxupload:connection-site_url}
#dash_path = ${dash:location}/bin/dash
#curl_path = ${curl:location}/bin/curl
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/instance.cfg 0000664 0000000 0000000 00000000407 13712426147 0027640 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-davstorage:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/software.cfg 0000664 0000000 0000000 00000001671 13712426147 0027672 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/git/buildout.cfg
common.cfg
parts +=
slapos.cookbook-repository
slapos.toolbox-repository
check-recipe
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
branch = davstorage-ajaxplorer
git-executable = ${git:location}/bin/git
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = master
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[versions]
openssl = 1.0.1c
slapos.recipe.template = 2.4.2
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/davstorage/todo.rst 0000664 0000000 0000000 00000000424 13712426147 0027051 0 ustar 00root root 0000000 0000000 TODO:
+ Move Ajaxplorer to a separate directory (currently in the www/ directory
accessed by webdav client, and so might delete files)
+ Make configuration part disappear ! (configuration when instance deploy,
no need for user to do it at each start of a new instance)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/ 0000775 0000000 0000000 00000000000 13712426147 0024303 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/buildout.hash.cfg 0000664 0000000 0000000 00000000114 13712426147 0027531 0 ustar 00root root 0000000 0000000 [instance]
filename = instance.cfg
md5sum = 646e50cfa93681e8bd85767621c7a39d slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/instance.cfg 0000664 0000000 0000000 00000004431 13712426147 0026572 0 ustar 00root root 0000000 0000000 [buildout]
parts =
dream_simulation
dream_platform
dream_test_suite
dream_interpreter
grunt_watch
publish-connection-parameter
dream-platform-url-available
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
extends = ${monitor-template:rendered}
# parameters
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[dream_platform_parameter]
port = 18080
host = $${instance-parameter:ipv6-random}
url = http://[$${:host}]:$${:port}
# interpreter
[dream_interpreter]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/dream_interpreter
wrapper-path = $${buildout:bin-directory}/dream_interpreter
# service
[dream_platform]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/dream_platform --debug --host $${dream_platform_parameter:host} --port $${dream_platform_parameter:port} --log $${directory:log}/dream_platform.log
wrapper-path = $${directory:service}/dream_platform
[dream-platform-url-available]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url= $${dream_platform_parameter:url}
[grunt_watch]
recipe = slapos.cookbook:wrapper
command-line = bash -c 'cd ${dream-repository.git:location}; PATH=${nodejs:location}/bin/:$PATH ${dream-repository.git:location}/node_modules/grunt-cli/bin/grunt watch -f > $${directory:log}/grunt.log'
wrapper-path = $${directory:service}/dream_grunt_watch
# CLI
[dream_simulation]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/dream_simulation
wrapper-path = $${directory:script}/dream_simulation
[dream_test_suite]
recipe = slapos.cookbook:wrapper
command-line = ${dream_testrunner:script}
wrapper-path = $${directory:script}/dream_test_suite
[directory]
recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
etc = $${:home}/etc
var = $${:home}/var
script = $${:etc}/run/
service = $${:etc}/service
promise = $${:etc}/promise/
log = $${:var}/log
[publish-connection-parameter]
recipe = slapos.cookbook:publishurl
url = $${dream_platform_parameter:url}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/software.cfg 0000664 0000000 0000000 00000002267 13712426147 0026625 0 ustar 00root root 0000000 0000000 [buildout]
versions = versions
extends =
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../component/manpy/buildout.cfg
../../stack/monitor/buildout.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
manpy
dream_testrunner
npm_install
instance
[gcc]
# Always build GCC for Fortran (see openblas).
max_version = 0
[instance]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance.cfg
[dream_testrunner]
recipe = zc.recipe.testrunner
eggs = dream
script = dream_testrunner
initialization =
${manpy:initialization}
[npm_install]
recipe = plone.recipe.command
stop-on-error = true
command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
update_command = ${:command}
[versions]
rpy2 = 2.4.0
pydot = 1.0.28
xlrd = 0.9.3
xlwt = 0.7.5
numpy = 1.16.4
scipy = 0.13.3
simpy = 3.0.5
zope.dottedname = 4.1.0
tablib = 0.10.0
mysqlclient = 1.3.12
# indirect dependancies
cp.recipe.cmd = 0.5
plone.recipe.command = 1.1
slapos.recipe.template = 4.4
zope.exceptions = 4.0.7
zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0
zope.testrunner = 4.4.6
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/software_testnode.cfg 0000664 0000000 0000000 00000000477 13712426147 0030533 0 ustar 00root root 0000000 0000000 [buildout]
extends =
software.cfg
# nodejs installation script does not support too deep directory structure like
# we can have when installing testnode in a webrunner. Since we do not need
# nodejs for the simulation executor, we can simply disable this section
[nodejs]
recipe =
location =
[npm_install]
recipe =
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/test/ 0000775 0000000 0000000 00000000000 13712426147 0025262 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/test/README.md 0000664 0000000 0000000 00000000041 13712426147 0026534 0 ustar 00root root 0000000 0000000 Tests for DREAM software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/test/setup.py 0000664 0000000 0000000 00000003704 13712426147 0027000 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.dream'
with open("README.md") as f:
long_description = f.read()
setup(name=name,
version=version,
description="Test for SlapOS' DREAM",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'requests'
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/dream/test/test.py 0000664 0000000 0000000 00000003467 13712426147 0026625 0 ustar 00root root 0000000 0000000 ##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, DREAMTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestHTTPAccess(DREAMTestCase):
def setUp(self):
self.url = self.computer_partition.getConnectionParameterDict()["url"]
def test(self):
self.assertEqual(
requests.codes.ok,
requests.get(self.url, verify=False).status_code
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/ 0000775 0000000 0000000 00000000000 13712426147 0024066 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/README.rst 0000664 0000000 0000000 00000007310 13712426147 0025556 0 ustar 00root root 0000000 0000000 Available ``software-type`` values
==================================
- ``default``
Recommended for development and production use. Automatic creation of
erp5-site.
Notes
=====
This software release is not intended to be accessed directly, but through a
front-end instance which is expected to contains the RewriteRules_ (or
equivalent) needed to relocate Zope's urls via its VirtualHostMonster_. See the
``frontend`` erp5 instance parameter.
ERP5 defaults connect to the public cloudooo on https://cloudooo.erp5.net/.
See the ``cloudooo`` Software Release to setup a cloudooo cluster if necessary.
Replication
===========
Replication allows setting up an ERP5 instance whose data follows another
instance.
Relations between ERP5 instances in a replication graph depend in what is
supported by individual data managers (ex: a neo cluster can replicate from a
neo cluster which itself replicates from a 3rd).
Replication lag constraints (aka sync/async replication) depends on individual
data managers (ex: neo replication between clusters is always asynchronous).
Ignoring replication lag, replicated data can be strictly identical (ex:
replicating ZODB or SQL database will contain the same data as upstream), or
may imply some remaping (ex: replicating Zope logs from an instance with 2 zope
families with 2 partition of 2 zopes each to an instance with a single zope
total).
Data whose replication is supported
-----------------------------------
- neo database
Data whose replication will eventually be supported
---------------------------------------------------
- mariadb database
- zope ``zope-*-access.log`` and ``zope-*-Z2.log``
- ``mariadb-slow.log``
Data whose replication is not planned
-------------------------------------
- zeo: use neo instead
Setting up replication
----------------------
In addition to your usual parameter set, you needs to provide the following parameters::
{
"zope-partition-dict": {}, So no zope is instanciated
"zodb": [
{
"storage-dict": {
"upstream-masters": ..., As published by to-become upstream ERP5 instance as "neo-masters"
},
"type": "neo", The only ZODB type supporting replication
...
}
...
]
...
}
Port ranges
===========
This software release assigns the following port ranges by default:
==================== ==========
Partition type Port range
==================== ==========
memcached-persistent 2000-2009
memcached-volatile 2010-2019
smtp 2025-2029
neo (admin, master) 2050-2052
mariadb 2099
zeo 2100-2149
balancer 2150-2199
zope 2200-*
jupyter 8888
caucase 8890,8891
==================== ==========
Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to
care about them as a user (but a Software Release developer needs to know
them).
Zope partitions should be assigned port ranges starting at 2200, incrementing
by some value which depends on how many zope process you want per partition
(see the ``port-base`` parameter in ``zope-partition-dict``).
Notes to the Software Release developer: These ranges are not strictly
defined. Not each port is actually used so one may reduce alread-assigned
ranges if needed (ex: memcached partitions use actually fewer ports). There
should be enough room for evolution (as between smtp and mariadb types). It is
important to not allocate any port after 2200 as user may have assigned ports
to his zope processes.
.. _RewriteRules: http://httpd.apache.org/docs/current/en/mod/mod_rewrite.html#rewriterule
.. _VirtualHostMonster: http://docs.zope.org/zope2/zope2book/VirtualHosting.html instance-erp5-input-schema.json 0000664 0000000 0000000 00000041365 13712426147 0031763 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5 {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Parameters to instantiate ERP5",
"additionalProperties": false,
"definitions": {
"tcpv4port": {
"$ref": "./schemas-definitions.json#/tcpv4port"
}
},
"properties": {
"sla-dict": {
"description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references (note: Zope partitions reference must be prefixed with \"zope-\").",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
},
"uniqueItems": true
},
"type": "object"
},
"site-id": {
"description": "ERP5Site object's id. An empty value disables automatic site creation.",
"default": "erp5",
"type": "string"
},
"bt5": {
"description": "Business Template to install at automatic site creation. By default, all configurators are installed.",
"type": "string"
},
"id-store-interval": {
"description": "Set Store Interval of default SQL Non Continuous Increasing Id Generator at automatic site creation. If unset, the value from the erp5_core Business Template is not touched.",
"type": "integer"
},
"timezone": {
"description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)",
"default": "UTC",
"type": "string"
},
"deadlock-debugger-password": {
"description": "Password for /manage_debug_threads",
"type": "string"
},
"inituser-login": {
"description": "Login of the initial/rescue user",
"default": "zope",
"type": "string"
},
"inituser-password": {
"description": "Password of the initial/rescue user",
"type": "string"
},
"developer-list": {
"description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
"items": {
"pattern": "^\\S+$",
"type": "string"
},
"uniqueItems": true,
"type": "array"
},
"hostalias-dict": {
"description": "Hostname-to-domain-name mapping",
"default": {},
"additionalProperties": {
"description": "A hostname to which current entry will resolve",
"type": "string"
},
"type": "object"
},
"hosts-dict": {
"description": "Host entries to be used in addition to and/or overriding auto-generated ones (erp5-catalog-0, erp5-cloudooo, erp5-memcached-persistent, erp5-memcached-volatile and erp5-smtp)",
"patternProperties": {
".*": {
"description": "An IP or domain name to which current entry will resolve",
"type": "string"
}
},
"type": "object"
},
"frontend": {
"description": "Front-end slave instance request parameters",
"properties": {
"software-url": {
"description": "Front-end's software type. If this parameter is empty, no front-end instance is requested. Else, sla-dict must specify 'frontend' which is a special value matching all frontends (e.g. {\"instance_guid=bar\": [\"frontend\"]}).",
"default": "",
"type": "string",
"format": "uri"
},
"domain": {
"description": "The domain name to request front-end to respond as.",
"default": "",
"type": "string"
},
"software-type": {
"description": "Request a front-end slave instance of this software type.",
"default": "RootSoftwareInstance",
"type": "string"
},
"virtualhostroot-http-port": {
"description": "Front-end slave http port. Port where http requests to frontend will be redirected.",
"default": 80,
"type": "integer"
},
"virtualhostroot-https-port": {
"description": "Front-end slave https port. Port where https requests to frontend will be redirected.",
"default": 443,
"type": "integer"
}
},
"type": "object"
},
"wsgi": {
"description": "If set to true, Zope is run as a WSGI application, instead of using the Medusa HTTP server.",
"type": "boolean",
"default": true
},
"zope-partition-dict": {
"description": "Zope layout definition",
"default": {
"1": {}
},
"patternProperties": {
".*": {
"additionalProperties": false,
"properties": {
"family": {
"description": "The family this partition is part of. For example: 'public', 'admin', 'backoffice', 'web-service'... Each family gets its own balancer entry. It has no special meaning for the system.",
"default": "default",
"type": "string"
},
"instance-count": {
"description": "Number of Zopes to setup on this partition",
"default": 1,
"type": "integer"
},
"thread-amount": {
"description": "Number of worker threads for each created Zope process",
"default": 4,
"type": "integer"
},
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"default": 5,
"type": "number"
},
"private-dev-shm": {
"description": "Size of private /dev/shm for wendelin.core. If sysctl kernel.unprivileged_userns_clone exists, it must be set to 1.",
"type": "string"
},
"ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.",
"description": "If set to true, will set SSL Client verification to required on apache VirtualHost which allow to access this zope instance.",
"type": "boolean",
"default": false
},
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled). Mixing webdav and non-webdav nodes in a single family will give unspecified results.",
"default": false,
"type": "boolean"
},
"longrequest-logger-interval": {
"description": "Period, in seconds, with which LongRequestLogger polls worker thread stack traces, or -1 to disable",
"default": -1,
"type": "integer"
},
"longrequest-logger-timeout": {
"description": "Transaction duration after which LongRequestLogger will start logging its stack trace, in seconds",
"default": 1,
"type": "integer"
},
"large-file-threshold": {
"description": "Requests bigger than this size get saved into a temporary file instead of being read completely into memory, in bytes",
"default": "10MB",
"type": "string"
},
"port-base": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
},
{
"description": "Start allocating ports at this value. Useful if one needs to make several partitions share the same port range (ie, several partitions bound to a single address)",
"default": 2200
}
]
}
},
"type": "object"
}
},
"type": "object"
},
"kumofs": {
"description": "Persistent memcached service",
"allOf": [
{
"$ref": "./instance-kumofs-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2000
}
}
}
],
"type": "object"
},
"memcached": {
"description": "Volatile memcached service",
"allOf": [
{
"$ref": "./instance-kumofs-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2010
}
}
}
],
"type": "object"
},
"cloudooo-url": {
"description": "Format conversion service URL",
"pattern": "^https?://",
"type": "string",
"format": "uri"
},
"cloudooo-retry-count": {
"description": "Define retry count for cloudooo in network error case in test",
"type": "integer",
"default": 2
},
"smtp": {
"description": "Mail queuing and relay service",
"allOf": [
{
"$ref": "./instance-smtp-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2010
}
}
}
],
"type": "object"
},
"mariadb": {
"description": "Relational database service",
"allOf": [
{
"$ref": "./instance-mariadb-schema.json"
},
{
"properties": {
"tcpv4-port": {
"default": 2099
}
}
}
],
"type": "object"
},
"zodb-zeo": {
"description": "Common settings ZEO servers",
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "#/definitions/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward"
}
]
},
"backup-periodicity": {
"description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined). Enter 'never' to disable backups.",
"default": "daily",
"type": "string"
},
"tidstorage-repozo-path": {
"description": "Directory for backup timestamp and tidstorage status files.",
"default": "~/srv/backup/tidstorage",
"type": "string"
}
},
"type": "object"
},
"zodb": {
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
"items": {
"required": [
"type"
],
"properties": {
"name": {
"description": "Database name",
"default": "main",
"type": "string"
},
"mount-point": {
"description": "Mount point",
"default": "/",
"type": "string"
},
"storage-dict": {
"description": "Storage configuration. For NEO, 'logfile' is automatically set (see https://lab.nexedi.com/nexedi/neoppod/blob/master/neo/client/component.xml for other settings).",
"properties": {
"ssl": {
"description": "For external NEO. Pass false if you want to disable SSL or pass custom values for ca/cert/key.",
"default": true,
"type": "boolean"
}
},
"patternProperties": {
".!$": {
"$ref": "#/properties/zodb/items/patternProperties/.!$"
}
},
"additionalProperties": {
"$ref": "#/properties/zodb/items/additionalProperties"
},
"type": "object"
},
"type": true,
"server": true
},
"oneOf": [
{
"title": "zeo",
"properties": {
"type": {
"description": "Storage type",
"const": "zeo"
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. The partition reference is 'zodb'.",
"$ref": "./instance-zeo-schema.json"
}
}
},
{
"title": "neo",
"properties": {
"type": {
"description": "Storage type",
"const": "neo"
},
"server": {
"description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
"$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
}
}
}
],
"patternProperties": {
".!$": {
"description": "Override with the value of the first item whose zope id matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"type": [
"integer",
"string"
]
}
],
"type": "array"
},
"type": "array"
}
},
"additionalProperties": {
"type": [
"integer",
"string"
]
},
"type": "object"
},
"type": "array"
},
"jupyter": {
"description": "Jupyter subinstance parameters",
"properties": {
"enable": {
"description": "Whether to enable creation of associated Jupyter subinstance",
"default": false,
"type": "boolean"
},
"zope-family": {
"description": "Zope family to connect Jupyter to by default",
"default": "",
"type": "string"
}
},
"type": "object"
},
"wendelin-core-zblk-fmt": {
"description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
"default": "",
"type": "string"
},
"caucase": {
"description": "Caucase certificate authority parameters",
"properties": {
"url": {
"title": "Caucase URL",
"description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
"default": "",
"type": "string",
"format": "uri"
}
},
"additionalProperties": {
"$ref": "../caucase/instance-caucase-input-schema.json"
},
"type": "object"
},
"test-runner": {
"description": "Test runner parameters.",
"properties": {
"enabled": {
"description": "Generate helper scripts to run test suite.",
"default": true,
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
"extra-database-count": {
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
}
},
"type": "object"
},
"balancer": {
"description": "HTTP(S) load balancer proxy parameters",
"properties": {
"ssl": {
"description": "HTTPS certificate generation parameters",
"properties": {
"frontend-caucase-url-list": {
"title": "Frontend Caucase URL List",
"description": "List of URLs of caucase service of frontend groups to authenticate access from them.",
"type": "array",
"items": {
"type": "string",
"format": "uri"
},
"uniqueItems": true
},
"caucase-url": {
"title": "Caucase URL",
"description": "URL of caucase service to use. If not set, global setting will be used.",
"type": "string",
"format": "uri"
},
"csr": {
"title": "csr",
"description": "PEM-encoded certificate signature request to request server certificate with. If not provided, HTTPS will be disabled.",
"type": "string"
},
"max-crl-update-delay": {
"title": "Periodicity of CRL update (days)",
"description": "CRL will be updated from caucase at least this often.",
"type": "number",
"default": 1.0
}
},
"type": "object"
}
},
"type": "object"
}
}
}
instance-erp5-output-schema.json 0000664 0000000 0000000 00000004663 13712426147 0032164 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5 {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by ERP5 instantiation",
"additionalProperties": false,
"properties": {
"hosts-dict": {
"description": "Hosts mapping, including auto-generated entries",
"patternProperties": {
".*": {
"description": "IP current entry resolves to",
"type": "string"
}
},
"type": "object"
},
"site-id": {
"description": "Chosen ERP5Site object identifier",
"type": "string"
},
"inituser-login": {
"description": "Initial user login",
"type": "string"
},
"inituser-password": {
"description": "Initial user password",
"type": "string"
},
"deadlock-debugger-password": {
"description": "Deadlock debugger password",
"type": "string"
},
"memcached-persistent-url": {
"description": "Persistent memcached access information",
"pattern": "^memcached://",
"type": "string"
},
"memcached-volatile-url": {
"description": "Volatile memcached access information",
"pattern": "^memcached://",
"type": "string"
},
"mariadb-database-list": {
"description": "Relational database access information",
"items": {
"pattern": "^mysql://",
"type": "string"
},
"uniqueItems": true,
"type": "array"
},
"mariadb-test-database-list": {
"description": "Relational database access information",
"items": {
"pattern": "^mysql://",
"type": "string"
},
"uniqueItems": true,
"type": "array"
},
"neo-masters": {
"$ref": "../neoppod/instance-neo-output-schema.json#/properties/masters"
},
"neo-admins": {
"$ref": "../neoppod/instance-neo-output-schema.json#/properties/admins"
},
"jupyter-url": {
"description": "Jupyter notebook web UI access information",
"pattern": "^https://",
"type": "string"
},
"caucase-http-url": {
"description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.",
"pattern": "^http://",
"type": "string"
}
},
"patternProperties": {
"family-.*": {
"description": "Zope family access information",
"pattern": "^https://",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/instance-kumofs-schema.json 0000664 0000000 0000000 00000001051 13712426147 0031322 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": [
"tcpv4-port"
],
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward"
}
]
},
"ram-storage-size": {
"description": "If 0 use disk storage, otherwise use ram and limit data size to this many megabytes",
"default": 0,
"type": "integer"
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/instance-mariadb-schema.json 0000664 0000000 0000000 00000011422 13712426147 0031420 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going downward"
}
]
},
"database-list": {
"description": "Databases to create and respective user credentials getting all privileges on it",
"default": [
{
"name": "erp5",
"user": "user",
"password": "insecure"
}
],
"minItems": 1,
"items": {
"required": [
"name",
"user",
"password"
],
"properties": {
"name": {
"description": "Database name",
"type": "string"
},
"user": {
"description": "User name",
"type": "string"
},
"password": {
"description": "User password",
"type": "string"
}
},
"type": "object"
},
"type": "array"
},
"catalog-backup": {
"description": "Backup control knobs",
"properties": {
"full-retention-days": {
"description": "How many days full backups must be retained, -1 meaning full backups are disabled and 0 meaning no expiration",
"default": 7,
"minimum": -1,
"type": "integer"
},
"incremental-retention-days": {
"description": "How many days incremental backups (binlogs) must be retained, -1 meaning incremental backups are disabled and 0 meaning no expiration, defaulting to full-retention-days' value",
"minimum": -1,
"type": "integer"
}
},
"type": "object"
},
"backup-periodicity": {
"description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined).",
"default": "daily",
"type": "string"
},
"innodb-buffer-pool-size": {
"description": "See MariaDB documentation on innodb_buffer_pool_size",
"minimum": 0,
"type": "integer"
},
"innodb-buffer-pool-instances": {
"description": "See MariaDB documentation on innodb_buffer_pool_instances",
"minimum": 1,
"type": "integer"
},
"innodb-log-file-size": {
"description": "See MariaDB documentation on innodb_log_file_size",
"minimum": 0,
"type": "integer"
},
"innodb-log-buffer-size": {
"description": "See MariaDB documentation on innodb_log_buffer_size",
"minimum": 0,
"type": "integer"
},
"innodb-file-per-table": {
"description": "See MariaDB documentation on innodb_file_per_table",
"minimum": 0,
"maximum": 1,
"default": 0,
"type": "integer"
},
"long-query-time": {
"description": "Number of seconds above which long queries are logged",
"minimum": 0,
"default": 1,
"type": "number"
},
"max-connection-count": {
"description": "See MariaDB documentation on max_connections. If not provided, a value suitable for the number of request Zope processes is chosen.",
"minimum": 0,
"type": "integer"
},
"relaxed-writes": {
"description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
"default": false,
"type": "boolean"
},
"ssl": {
"description": "Enable and define SSL support for network connections",
"default": {},
"properties": {
"ca-crt": {
"description": "Certificate Authority's certificate, in PEM format",
"type": "string"
},
"crt": {
"description": "Server's certificate, in PEM format (mandatory to enable SSL support)",
"type": "string"
},
"key": {
"description": "Server's key, in PEM format (mandatory to enable SSL support)",
"type": "string"
},
"crl": {
"description": "Server's certificate revocation list, in PEM format",
"type": "string"
},
"cipher": {
"description": "Permissible cipher specifications, separated by colons",
"type": "string"
}
},
"type": "object"
},
"odbc-ini": {
"description": "Contents of odbc.ini file, see unixodbc document",
"default": "",
"type": "string"
},
"environment-variables": {
"description": "Extra environment variables for mysqld may be required to use third party ODBC libraries for CONNECT storage engine.",
"items": {
"type": "string"
},
"type": "array"
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/instance-smtp-schema.json 0000664 0000000 0000000 00000003436 13712426147 0031012 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"required": [
"tcpv4-port"
],
"type": "object",
"properties": {
"tcpv4-port": {
"allOf": [
{
"$ref": "./schemas-definitions.json#/tcpv4port"
},
{
"description": "Start allocating ports at this value, going upward"
}
]
},
"postmaster": {
"description": "Mail address to send technical mails to. Non-empty value required for smptd relay service to be deployed. Values will be put in alias-dict as 'postmaster' key (alias-dict takes precedence)",
"default": "",
"type": "string"
},
"alias-dict": {
"description": "Mail alias support",
"default": {},
"patternProperties": {
".*": {
"description": "List of addresses alias expands to",
"type": "array"
}
},
"type": "object"
},
"relay": {
"description": "Forward outgoing mails to a specific relay. If enabled, relay must support TLS-encrypted SASL authentication.",
"dependencies": {
"host": [
"sasl-credential"
]
},
"properties": {
"host": {
"description": "Host name or address of relay, with optional port (ex: '[example.com]:submission'). Enclosing hostname with [] prevents MX lookup.",
"type": "string"
},
"sasl-credential": {
"description": "SASL credential, in the login:password form",
"type": "string"
}
},
"default": {},
"type": "object"
},
"divert": {
"description": "Intercept all mails and send them to given addresses instead of original recipient",
"type": "array",
"items": {
"type": "string"
},
"uniqueItems": true
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/instance-zeo-schema.json 0000664 0000000 0000000 00000001605 13712426147 0030620 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"additionalProperties": false,
"properties": {
"backup": {
"description": "'%(backup)s' is expanded to partition's ZODB backup path (typically 'srv/backup/zodb'), and %(name)s with the export id",
"default": "%(backup)s/%(name)s",
"type": "string"
},
"family": {
"description": "Opaque name used to regroup/separate mountpoints under different ZEO processes (must be valid as a file name and as a ConfigParser section name)",
"default": "default",
"pattern": "^[^<>:\"/\\|?*\\]\\[ ]*$",
"type": "string"
},
"path": {
"description": "FileStorage file path, '%(zodb)s' occurrences are replaced with the path to partition's srv/zodb directory, and %(name)s with the export id",
"default": "%(zodb)s/%(name)s.fs",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/schemas-definitions.json 0000664 0000000 0000000 00000000217 13712426147 0030715 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-07/schema#",
"tcpv4port": {
"minimum": 0,
"maximum": 65535,
"type": "integer"
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/software-shared.cfg 0000664 0000000 0000000 00000000213 13712426147 0027641 0 ustar 00root root 0000000 0000000 [buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/software.cfg 0000664 0000000 0000000 00000000065 13712426147 0026402 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/erp5/buildout.cfg
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/software.cfg.json 0000664 0000000 0000000 00000000613 13712426147 0027351 0 ustar 00root root 0000000 0000000 {
"name": "ERP5",
"description": "ERP5, Open-Source ERP",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"description": "No automated database modification (ERP5Site is not automatically created).",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 0
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/ 0000775 0000000 0000000 00000000000 13712426147 0025045 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/README.md 0000664 0000000 0000000 00000000040 13712426147 0026316 0 ustar 00root root 0000000 0000000 Tests for ERP5 software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/setup.py 0000664 0000000 0000000 00000004014 13712426147 0026556 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.erp5'
with open("README.md") as f:
long_description = f.read()
setup(name=name,
version=version,
description="Test for SlapOS' ERP5 software releae",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'supervisor',
'slapos.libnetworkcache',
'erp5.util',
'psutil',
'requests',
'mysqlclient',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/test/ 0000775 0000000 0000000 00000000000 13712426147 0026024 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/test/__init__.py 0000664 0000000 0000000 00000005247 13712426147 0030145 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import json
import os
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
setup_module_executed = False
def setUpModule():
# slapos.testing.testcase's only need to be executed once
global setup_module_executed
if not setup_module_executed:
_setUpModule()
setup_module_executed = True
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
# ERP5 instanciation needs to run several times before being ready, as
# the root instance request more instances.
instance_max_retry = 7 # XXX how many times ?
def getRootPartitionConnectionParameterDict(self):
"""Return the output paramters from the root partition"""
return json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/test/test_balancer.py 0000664 0000000 0000000 00000024436 13712426147 0031215 0 ustar 00root root 0000000 0000000 from . import ERP5InstanceTestCase
from . import setUpModule
from slapos.testing.utils import findFreeTCPPort
from BaseHTTPServer import HTTPServer
from BaseHTTPServer import BaseHTTPRequestHandler
import OpenSSL.SSL
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography import x509
from cryptography.x509.oid import NameOID
import hashlib
import json
import multiprocessing
import os
import requests
import shutil
import subprocess
import tempfile
import time
setUpModule # pyflakes
class TestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = {
'Path': self.path,
'Incoming Headers': self.headers.dict
}
response = json.dumps(response, indent=2)
self.end_headers()
self.wfile.write(response)
class TestFrontendXForwardedFor(ERP5InstanceTestCase):
__partition_reference__ = 'xff'
http_server_process = None
frontend_caucase_dir = None
frontend_caucased_process = None
backend_caucase_dir = None
backend_caucased_process = None
@classmethod
def getInstanceSoftwareType(cls):
return 'balancer'
@classmethod
def setUpClass(cls):
# start a dummy web server echoing headers.
http_server_port = findFreeTCPPort(cls._ipv4_address)
server = HTTPServer(
(cls._ipv4_address, http_server_port),
TestHandler)
cls.http_server_process = multiprocessing.Process(
target=server.serve_forever, name='HTTPServer')
cls.http_server_process.start()
cls.http_server_netloc = '%s:%s' % (cls._ipv4_address, http_server_port)
# start a caucased and generate a valid client certificate.
cls.computer_partition_root_path = os.path.abspath(os.curdir)
cls.frontend_caucase_dir = tempfile.mkdtemp()
frontend_caucased_dir = os.path.join(cls.frontend_caucase_dir, 'caucased')
os.mkdir(frontend_caucased_dir)
frontend_user_dir = os.path.join(cls.frontend_caucase_dir, 'user')
os.mkdir(frontend_user_dir)
frontend_service_dir = os.path.join(cls.frontend_caucase_dir, 'service')
os.mkdir(frontend_service_dir)
frontend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
cls.frontend_caucased_url = 'http://' + frontend_caucased_netloc
cls.user_certificate = frontend_user_key = os.path.join(frontend_user_dir, 'client.key.pem')
frontend_user_csr = os.path.join(frontend_user_dir, 'client.csr.pem')
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(frontend_user_key, 'wb') as f:
f.write(key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'user'),
])).sign(key, hashes.SHA256(), default_backend())
with open(frontend_user_csr, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
cls.software_release_root_path = os.path.join(
cls.slap._software_root,
hashlib.md5(cls.getSoftwareURL()).hexdigest(),
)
caucased_path = os.path.join(cls.software_release_root_path, 'bin', 'caucased')
caucase_path = os.path.join(cls.software_release_root_path, 'bin', 'caucase')
cls.frontend_caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(frontend_caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(frontend_caucased_dir, 'server.key.pem'),
'--netloc', frontend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(10):
try:
if requests.get(cls.frontend_caucased_url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError, 'caucased failed to start.'
cau_args = [
caucase_path,
'--ca-url', cls.frontend_caucased_url,
'--ca-crt', os.path.join(frontend_user_dir, 'service-ca-crt.pem'),
'--crl', os.path.join(frontend_user_dir, 'service.crl'),
'--user-ca-crt', os.path.join(frontend_user_dir, 'user-ca-crt.pem'),
'--user-crl', os.path.join(frontend_user_dir, 'user.crl'),
]
cas_args = [
caucase_path,
'--ca-url', cls.frontend_caucased_url,
'--ca-crt', os.path.join(frontend_service_dir, 'service-ca-crt.pem'),
'--crl', os.path.join(frontend_service_dir, 'service.crl'),
'--user-ca-crt', os.path.join(frontend_service_dir, 'user-ca-crt.pem'),
'--user-crl', os.path.join(frontend_service_dir, 'user.crl'),
]
caucase_process = subprocess.Popen(
cau_args + [
'--mode', 'user',
'--send-csr', frontend_user_csr,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
result = caucase_process.communicate()
print result
csr_id = result[0].split()[0]
subprocess.check_call(
cau_args + [
'--mode', 'user',
'--get-crt', csr_id, frontend_user_key,
],
)
cls.client_certificate = frontend_service_key = os.path.join(frontend_service_dir, 'crt.pem')
frontend_service_csr = os.path.join(frontend_service_dir, 'csr.pem')
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(frontend_service_key, 'wb') as f:
f.write(key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'service'),
])).sign(key, hashes.SHA256(), default_backend())
with open(frontend_service_csr, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
caucase_process = subprocess.Popen(
cas_args + [
'--send-csr', frontend_service_csr,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
result = caucase_process.communicate()
csr_id = result[0].split()[0]
for _ in range(10):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, frontend_service_key,
],
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError, 'getting service certificate failed.'
# start a caucased and server certificate.
cls.backend_caucase_dir = tempfile.mkdtemp()
backend_caucased_dir = os.path.join(cls.backend_caucase_dir, 'caucased')
os.mkdir(backend_caucased_dir)
backend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
cls.backend_caucased_url = 'http://' + backend_caucased_netloc
cls.backend_caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(backend_caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(backend_caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(10):
try:
if requests.get(cls.backend_caucased_url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError, 'caucased failed to start.'
super(TestFrontendXForwardedFor, cls).setUpClass()
@classmethod
def getInstanceParameterDict(cls):
return {
'_': json.dumps({
'tcpv4-port': 3306,
'computer-memory-percent-threshold': 100,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
'apachedex-configuration': '',
'apachedex-promise-threshold': 100,
'haproxy-server-check-path': '/',
'zope-family-dict': {
'default': ['dummy_http_server'],
'default-auth': ['dummy_http_server'],
},
'dummy_http_server': [[cls.http_server_netloc, 1, False]],
'backend-path-dict': {
'default': '/',
'default-auth': '/',
},
'ssl-authentication-dict': {
'default': False,
'default-auth': True,
},
'ssl': {
'caucase-url': cls.backend_caucased_url,
'frontend-caucase-url-list': [cls.frontend_caucased_url],
},
})
}
@classmethod
def _cleanup(cls, snapshot_name):
if cls.http_server_process:
cls.http_server_process.terminate()
if cls.frontend_caucased_process:
cls.frontend_caucased_process.terminate()
if cls.frontend_caucase_dir:
shutil.rmtree(cls.frontend_caucase_dir)
if cls.backend_caucased_process:
cls.backend_caucased_process.terminate()
if cls.backend_caucase_dir:
shutil.rmtree(cls.backend_caucase_dir)
super(TestFrontendXForwardedFor, cls)._cleanup(snapshot_name)
def test_x_forwarded_for_added_when_verified_connection(self):
for backend in ('default', 'default-auth'):
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])[backend]
result = requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
cert=self.client_certificate,
verify=False,
).json()
self.assertEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
def test_x_forwarded_for_stripped_when_not_verified_connection(self):
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
result = requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
verify=False,
).json()
self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
with self.assertRaises(OpenSSL.SSL.Error):
requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
verify=False,
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/test/test_erp5.py 0000664 0000000 0000000 00000025554 13712426147 0030323 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import glob
import urlparse
import socket
import time
import psutil
import requests
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
# What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable, sometimes the first request is 404, so we
# retry in a loop.
# If we can move the "create site" in slapos node instance, then this retry loop
# would not be necessary.
for i in range(1, 60):
r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
if r.status_code in (requests.codes.service_unavailable,
requests.codes.not_found):
delay = i * 2
self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay)
continue
if r.status_code != requests.codes.ok:
r.raise_for_status()
break
self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self):
"""Tests the IPv6 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instanciated with no parameters
"""
__partition_reference__ = 'defp'
class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Medusa server
"""
__partition_reference__ = 'medusa'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestApacheBalancerPorts(ERP5InstanceTestCase):
"""Instanciate with two zope families, this should create for each family:
- a balancer entry point with corresponding haproxy
- a balancer entry point for test runner
"""
__partition_reference__ = 'ap'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
"zope-partition-dict": {
"family1": {
"instance-count": 3,
"family": "family1"
},
"family2": {
"instance-count": 5,
"family": "family2"
},
},
})
}
def checkValidHTTPSURL(self, url):
parsed = urlparse.urlparse(url)
self.assertEqual(parsed.scheme, 'https')
self.assertTrue(parsed.hostname)
self.assertTrue(parsed.port)
def test_published_family_parameters(self):
# when we request two families, we have two published family-{family_name} URLs
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)])
self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format(family_name=family_name)])
def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per family
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[
'{family_name}-test-runner-url-list'.format(family_name=family_name)]
self.assertEqual(3, len(family_test_runner_url_list))
for url in family_test_runner_url_list:
self.checkValidHTTPSURL(url)
def test_zope_listen(self):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.assertEqual(
3 + 5,
len([p for p in all_process_info if p['name'].startswith('zope-')]))
def test_apache_listen(self):
# We have 2 families, apache should listen to a total of 3 ports per family
# normal access on ipv4 and ipv6 and test runner access on ipv4 only
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
sorted([
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
]))
def test_haproxy_listen(self):
# There is one haproxy per family
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [
p for p in all_process_info if p['name'].startswith('haproxy-')
]
haproxy_process = psutil.Process(process_info['pid'])
self.assertEqual([socket.AF_INET, socket.AF_INET], [
c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
])
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instanciated without test runner.
"""
__partition_reference__ = 'distr'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'test-runner': {'enabled': False}})}
def test_no_runUnitTestScript(self):
"""No runUnitTest script should be generated in any partition.
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
@classmethod
def getInstanceParameterDict(cls):
# The following example includes the most commonly used options,
# but not necessarily in a meaningful way.
return {'_': json.dumps({
"zodb": [{
"type": "zeo",
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
"storage-dict": {
"cache-size!": [
("a-.*", "50MB"),
],
},
}],
"zope-partition-dict": {
"a": {
"instance-count": 3,
},
"bb": {
"instance-count": 5,
"port-base": 2300,
},
},
})}
def test_zope_conf(self):
zeo_addr = json.loads(
self.getComputerPartition('zodb').getConnectionParameter('_')
)["storage-dict"]["root"]["server"]
def checkParameter(line, kw):
k, v = line.split()
self.assertFalse(k.endswith('!'), k)
try:
expected = kw.pop(k)
except KeyError:
if k == 'server':
return
self.assertIsNotNone(expected)
self.assertEqual(str(expected), v)
def checkConf(zodb, storage):
zodb["mount-point"] = "/"
zodb["pool-size"] = 4
zodb["pool-timeout"] = "10m"
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
i = conf.index("") + 1
conf = iter(conf[i:conf.index("", i)])
for line in conf:
if line == '':
for line in conf:
if line == '':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5/test/test/test_mariadb.py 0000664 0000000 0000000 00000022054 13712426147 0031037 0 ustar 00root root 0000000 0000000 ##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import glob
import urlparse
import socket
import time
import contextlib
import datetime
import MySQLdb
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
class MariaDBTestCase(ERP5InstanceTestCase):
"""Base test case for mariadb tests.
"""
__partition_reference__ = 'm'
@classmethod
def getInstanceSoftwareType(cls):
return "mariadb"
@classmethod
def _getInstanceParameterDict(cls):
return {
'tcpv4-port': 3306,
'max-connection-count': 5,
'max-slowqueries-threshold': 5,
'slowest-query-threshold': 10,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
# XXX should probably not be needed here
'computer-memory-percent-threshold': 100,
}
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps(cls._getInstanceParameterDict())}
def getDatabaseConnection(self):
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
database_name = db_url.path[1:]
return MySQLdb.connect(
user=db_url.username,
passwd=db_url.password,
host=db_url.hostname,
port=db_url.port,
db=database_name,
)
class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
CREATE TABLE test_utf8_collation (
col1 CHAR(10)
)
""")
cnx.store_result()
cnx.query(
"""
insert into test_utf8_collation values ("à"), ("あ")
""")
cnx.store_result()
cnx.query(
"""
select * from test_utf8_collation where col1 = "a"
""")
self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
class TestMroonga(MariaDBTestCase):
def test_mroonga_plugin_loaded(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("show plugins")
plugins = cnx.store_result().fetch_row(maxrows=1000)
self.assertIn(
('Mroonga', 'ACTIVE', 'STORAGE ENGINE', 'ha_mroonga.so', 'GPL'),
plugins)
def test_mroonga_normalize_udf(self):
# example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
self.assertEqual((('abcdあぃうぇリットル',),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
# this example fail with:
# OperationalError: (1123, "Can't initialize function 'mroonga_normalize'; mroonga_normalize(): nonexistent normalizer NormalizerMySQLUnicodeCIExceptKanaCI")
# same error on mroonga "official" docker images using mysql
# https://hub.docker.com/layers/groonga/mroonga/latest/images/sha256-e5a979801c95544ca3a1228d2c4d819820850e0162649553f2e94850e5e1c988?context=explore
# so it's probably OK to ignore
cnx.query(
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual((('ABCDあぃうぇ㍑',),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SET NAMES utf8")
cnx.store_result()
cnx.query(
"""
CREATE TABLE diaries (
day DATE PRIMARY KEY,
content VARCHAR(64) NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto"'
) Engine=Mroonga DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
cnx.store_result()
cnx.query(
"""INSERT INTO diaries VALUES ("2013-04-23", "ブラックコーヒーを飲んだ。")""")
cnx.store_result()
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ふらつく" IN BOOLEAN MODE)
""")
self.assertEqual((), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ブラック" IN BOOLEAN MODE)
""")
self.assertEqual(
((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
cnx.store_result().fetch_row(maxrows=2),
)
def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
# Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
CREATE TABLE `full_text` (
`uid` BIGINT UNSIGNED NOT NULL,
`SearchableText` MEDIUMTEXT,
PRIMARY KEY (`uid`),
FULLTEXT `SearchableText` (`SearchableText`) COMMENT 'parser "TokenBigramSplitSymbolAlphaDigit"'
) ENGINE=mroonga
""")
cnx.store_result()
cnx.query(
"""
INSERT INTO full_text VALUES
(1, "Gabriel Fauré Quick brown fox jumps over the lazy dog"),
(2, "武者小路 実篤 Slow white fox jumps over the diligent dog."),
(3, "( - + )")""")
cnx.store_result()
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ Faure' IN BOOLEAN MODE)
""")
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ 武者' IN BOOLEAN MODE)
""")
self.assertEqual(((2,),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ +quick +fox +dog' IN BOOLEAN MODE)
""")
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_stem(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT mroonga_command('register token_filters/stem')")
self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
CREATE TABLE memos (
id INT NOT NULL PRIMARY KEY,
content TEXT NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto", token_filters "TokenFilterStem"'
) Engine=Mroonga DEFAULT CHARSET=utf8
""")
cnx.store_result()
cnx.query(
"""INSERT INTO memos VALUES (1, "I develop Groonga"), (2, "I'm developing Groonga"), (3, "I developed Groonga")"""
)
cnx.store_result()
cnx.query(
"""
SELECT *
FROM memos
WHERE MATCH (content) AGAINST ("+develops" IN BOOLEAN MODE)
""")
self.assertEqual([
(1, "I develop Groonga"),
(2, "I'm developing Groonga"),
(3, "I developed Groonga"),
], list(sorted(cnx.store_result().fetch_row(maxrows=4))))
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/ 0000775 0000000 0000000 00000000000 13712426147 0025634 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/buildout.hash.cfg 0000664 0000000 0000000 00000001556 13712426147 0031075 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
filename = instance-default.cfg
md5sum = 40364ff26e9284cea97a58f3cd8c75e3
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/instance-default.cfg 0000664 0000000 0000000 00000017573 13712426147 0031560 0 ustar 00root root 0000000 0000000 [buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
extends = ${monitor2-template:rendered}
parts =
testnode-service
certificate-authority
ca-shellinabox
ca-httpd-testnode
monitor-base
monitor-publish
testnode-frontend
resiliency-exclude-file
shellinabox-frontend-reload
promises
[monitor-publish]
recipe = slapos.cookbook:publish
url = $${shellinabox-frontend:url}
frontend-url = $${testnode-frontend:connection-secure_access}
[pwgen]
recipe = slapos.cookbook:generate.password
storage-path = $${buildout:directory}/.password
[testnode]
recipe = slapos.cookbook:erp5testnode
slapos-directory = $${directory:slapos}
working-directory = $${directory:testnode}
test-suite-directory = $${directory:test-suite}
shared-part-list = $${directory:shared}
proxy-host = $${slap-network-information:local-ipv4}
proxy-port = 5000
log-directory = $${directory:log}
srv-directory = $${rootdirectory:srv}
software-directory = $${directory:software}
run-directory = $${directory:run}
test-node-title = $${slap-parameter:test-node-title}
node-quantity = $${slap-parameter:node-quantity}
ipv4-address = $${slap-network-information:local-ipv4}
ipv6-address = $${slap-network-information:global-ipv6}
test-suite-master-url = $${slap-parameter:test-suite-master-url}
instance-dict = $${slap-parameter:instance-dict}
software-path-list = $${slap-parameter:software-path-list}
keep-log-days = $${slap-parameter:keep-log-days}
git-binary = ${git:location}/bin/git
slapos-binary = ${buildout:bin-directory}/slapos
testnode = ${buildout:bin-directory}/testnode
zip-binary = ${zip:location}/bin/zip
httpd-pid-file = $${basedirectory:run}/httpd.pid
httpd-lock-file = $${basedirectory:run}/httpd.lock
httpd-conf-file = $${rootdirectory:etc}/httpd.conf
httpd-wrapper = $${rootdirectory:bin}/httpd
httpd-port = 9080
httpd-software-access-port = 9081
httpd-ip = $${slap-network-information:global-ipv6}
httpd-log-directory = $${basedirectory:log}
httpd-software-directory = $${directory:software}
httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
httpd-key-file = $${rootdirectory:etc}/httpd-private.key
configuration-file = $${rootdirectory:etc}/erp5testnode.cfg
log-file = $${basedirectory:log}/erp5testnode.log
wrapper = $${buildout:bin-directory}/erp5testnode-service
# Binaries
apache-binary = ${apache:location}/bin/httpd
apache-modules-dir = ${apache:location}/modules
apache-mime-file = ${apache:location}/conf/mime.types
apache-htpasswd = ${apache:location}/bin/htpasswd
[testnode-service]
# wrapper over erp5testnode which restarts the service when configuration changed
recipe = slapos.cookbook:wrapper
wrapper-path = $${basedirectory:services}/erp5testnode
command-line = $${testnode:wrapper}
hash-files =
$${testnode:wrapper}
$${testnode:configuration-file}
[shell-environment]
shell = ${bash:location}/bin/bash
[shellinabox]
recipe = slapos.recipe.template:jinja2
# We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
socket = $${directory:run}/siab.sock
mode = 0700
rendered = $${basedirectory:services}/shellinaboxd
template = inline:
#!/bin/sh
exec ${shellinabox:location}/bin/shellinaboxd \
--unixdomain-only=$${:socket}:$(id -u):$(id -g):0600 \
--service "/:$(id -u):$(id -g):HOME:$${shell-environment:shell} -l"
[shellinabox-frontend-config]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template = inline:
https://$${:hostname}:$${:port} {
bind $${:ipv6}
tls self_signed
gzip
log stdout
errors stderr
proxy / unix:$${shellinabox:socket}
basicauth $${:username} $${:passwd} {
realm "Test Node $${testnode:test-node-title}"
/
}
}
ipv6 = $${slap-network-information:global-ipv6}
hostname = [$${:ipv6}]
port = 8080
username = testnode
passwd = $${pwgen:passwd}
cert-file = $${directory:shellinabox}/public.crt
key-file = $${directory:shellinabox}/private.key
url = https://$${:username}:$${:passwd}@$${:hostname}:$${:port}
[shellinabox-frontend]
recipe = slapos.cookbook:wrapper
wrapper-path = $${rootdirectory:bin}/$${:_buildout_section_name_}
command-line =
${caddy:output} -conf $${shellinabox-frontend-config:rendered} -pidfile $${:pidfile}
url = $${shellinabox-frontend-config:url}
hostname = $${shellinabox-frontend-config:ipv6}
port = $${shellinabox-frontend-config:port}
pidfile = $${basedirectory:run}/$${:_buildout_section_name_}.pid
[shellinabox-frontend-reload]
recipe = slapos.cookbook:wrapper
wrapper-path = $${basedirectory:services}/$${:_buildout_section_name_}
command-line =
${bash:location}/bin/bash -c
"kill -s USR1 $$(${coreutils:location}/bin/cat $${shellinabox-frontend:pidfile}) \
&& ${coreutils:location}/bin/sleep infinity"
hash-files =
$${shellinabox-frontend-config:rendered}
$${shellinabox-frontend:wrapper-path}
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${directory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${basedirectory:services}/ca
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[cadirectory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:ca-dir}/requests/
private = $${directory:ca-dir}/private/
certs = $${directory:ca-dir}/certs/
newcerts = $${directory:ca-dir}/newcerts/
crl = $${directory:ca-dir}/crl/
[ca-shellinabox]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
executable = $${shellinabox-frontend:wrapper-path}
wrapper = $${basedirectory:services}/shellinabox-frontend
key-file = $${shellinabox-frontend-config:key-file}
cert-file = $${shellinabox-frontend-config:cert-file}
[ca-httpd-testnode]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
executable = $${testnode:httpd-wrapper}
wrapper = $${basedirectory:services}/httpd
key-file = $${testnode:httpd-key-file}
cert-file = $${testnode:httpd-cert-file}
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
services = $${rootdirectory:etc}/service
run = $${rootdirectory:var}/run
[directory]
recipe = slapos.cookbook:mkdirectory
slapos = $${rootdirectory:srv}/slapos
testnode = $${rootdirectory:srv}/testnode
shared = $${rootdirectory:srv}/shared
test-suite = $${rootdirectory:srv}/test_suite
log = $${basedirectory:log}/testnode
run = $${basedirectory:run}/testnode
software = $${rootdirectory:srv}/software
shellinabox = $${rootdirectory:srv}/shellinabox
ca-dir = $${rootdirectory:srv}/ca
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = $${directory:srv}/exporter.exclude
[testnode-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Test Node Frontend $${testnode:test-node-title}
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-software-access-port}
config-https-only = true
#software-type = custom-personal
return = domain secure_access
[promises]
recipe =
instance-promises =
$${shellinabox-frontend-listen-promise:name}
[shellinabox-frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port}
[slap-parameter]
node-quantity = 1
test-suite-master-url =
instance-dict =
software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.154/software/seleniumrunner/software.cfg"]
keep-log-days = 15
instance-erp5testnode-input-schema.json 0000664 0000000 0000000 00000001515 13712426147 0035270 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"test-node-title": {
"title": "Test Node Title",
"description": "Name for the Test Node which will be used at ERP5.",
"type": "string"
},
"node-quantity": {
"title": "Quantity of Parallel runs",
"description": "QUantity of Parallel tests will be executed on this Node.",
"type": "integer"
},
"test-suite-master-url": {
"title": "Task Distribution URL",
"description": "Url for the task distributor master on portal_task_distribution",
"type": "string"
},
"keep-log-days": {
"title": "Retention of test logs",
"description": "Number of days to keep logs and snapshots",
"type": "number",
"default": 15
}
}
}
instance-output-schema.json 0000664 0000000 0000000 00000000574 13712426147 0033056 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Apache Frontend instanciation",
"properties": {
"password": {
"description": "Password to access shellinabox.",
"type": "string"
},
"url": {
"description": "IPv6 URL to access the shell in a box.",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/instance.cfg 0000664 0000000 0000000 00000000403 13712426147 0030116 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-default:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/software.cfg 0000664 0000000 0000000 00000002723 13712426147 0030153 0 ustar 00root root 0000000 0000000 [buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/zip/buildout.cfg
../../component/bash/buildout.cfg
../../component/caddy/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/shellinabox/buildout.cfg
../../component/pwgen/buildout.cfg
../../component/apache/buildout.cfg
../../stack/monitor/buildout.cfg
parts =
slapos-command
slapos-cookbook
template
eggs
zip
git
apache
[eggs]
recipe = zc.recipe.egg
eggs =
erp5.util[testnode]
${lxml-python:egg}
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
mode = 0644
[template-default]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default.cfg
output = ${buildout:directory}/template-default.cfg
mode = 0644
[versions]
slapos.recipe.template = 4.4
dnspython = 1.15.0
PyXML = 0.8.5
WebOb = 1.8.5
WebTest = 2.0.33
backports.functools-lru-cache = 1.5
soupsieve = 1.8
#gitdb2 = 2.0.4
#smmap2 = 2.0.4
waitress = 1.2.1
z3c.etestbrowser = 2.0.1
zope.testbrowser = 5.3.2
# Required by:
# zope.testbrowser==5.3.2
WSGIProxy2 = 0.4.6
# Required by:
# WebTest==2.0.33
beautifulsoup4 = 4.7.1
# Required by:
# zope.testbrowser==5.3.2
zope.cachedescriptors = 4.3.1
# Required by:
# zope.schema==4.9.3
zope.event = 4.4
# Required by:
# zope.testbrowser==5.3.2
zope.schema = 4.9.3
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/software.cfg.json 0000664 0000000 0000000 00000000533 13712426147 0031120 0 ustar 00root root 0000000 0000000 {
"name": "ERP5 Test Node",
"description": "ERP5 Test Node Runner",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Default setup for ERP5TestNode Instance.",
"request": "instance-erp5testnode-input-schema.json",
"response": "instance-output-schema.json"
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/ 0000775 0000000 0000000 00000000000 13712426147 0027665 5 ustar 00root root 0000000 0000000 deploy-test/ 0000775 0000000 0000000 00000000000 13712426147 0032057 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite README.rst 0000664 0000000 0000000 00000000613 13712426147 0033546 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test deply-test
==========
Introduction
------------
This is software release to run tests on VMs, but without direct access (like
ssh) to the VM.
It is supposed to be used as backend for ERP5TestNode.
Characteristics and limitations:
* partitions share the same user (as they access files directly)
* this SR will be installed and instantiated only from local file system
(like git clone)
buildout.hash.cfg 0000664 0000000 0000000 00000001620 13712426147 0035310 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[deploy-script-controller-script]
filename = deploy-script-controller
md5sum = 8288e59eb442c662544daffbf446a033
[template-deploy-test]
filename = instance.cfg.in
md5sum = b673c497abe5b3e7de4c7fc8b01aa3d0
deploy-script-controller 0000664 0000000 0000000 00000005476 13712426147 0036775 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test #!/bin/bash
# Script for controlling deploy script
#
# Assumption: deployment script returning with code 0
# run correctly and everything is done
#
# This script is run inside of KVM, by passing its
# URL with bootstrap-script-url
# The script configuration happens by passing
# it in text form with data-to-vm
#
# Format of data-to-vm is shell script:
# URL=\nWAITTIME=\nTRIES=
#
# Expected values in configuration:
# * URL - the url of the script to download and test
# * WAITTIME - waiting time, before next try
# * TRIES - amount of tries
# Possible TODOs:
# * post results on each try
# * use function + trap to assure posting on exit
LOG_FILE=/var/log/test-script-deployment.log
wget -O /tmp/test-script.cfg.$$ -q http://10.0.2.100/data
source /tmp/test-script.cfg.$$
if [ -z "$LOG_FILE" ] ; then
echo "Output log file is missing"
exit 1
fi
if [ -z "$URL" ] ; then
echo "URL is missing" >> $LOG_FILE 2>&1
exit 1
fi
if [ -z "$WAITTIME" ] ; then
echo "WAITTIME missing" >> $LOG_FILE 2>&1
exit 1
fi
if [ -z "$TRIES" ] ; then
echo "TRIES missing" >> $LOG_FILE 2>&1
exit 1
fi
DEPLOYMENT_SCRIPT=/tmp/test-script-deployment.bash.$$
wget -O $DEPLOYMENT_SCRIPT -q $URL
if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
echo "exit 1" > $DEPLOYMENT_SCRIPT
fi
function add_log ()
{
LOG_FILE=$1
for f in /opt/slapos/log/slapos-node-{software,instance}.log ; do
echo "Tail of '$f':" >> $LOG_FILE
tail -n 500 $f >> $LOG_FILE
done
}
function add_checks ()
{
LOG_FILE=$1
echo 'lsof -Pni' >> $LOG_FILE 2>&1
lsof -Pni >> $LOG_FILE 2>&1
echo 'iptables-save' >> $LOG_FILE 2>&1
iptables-save >> $LOG_FILE 2>&1
for f in /tmp/playbook-* ; do echo $f ; cat $f; echo; done >> $LOG_FILE 2>&1
echo 'slapos node status' >> $LOG_FILE 2>&1
slapos node status >> $LOG_FILE 2>&1
}
function upload ()
{
try=$1
LOG_FILE=$2
add_log $LOG_FILE
add_checks $LOG_FILE
t=`date '+%Y%m%d%H%S'`
mv $LOG_FILE ${LOG_FILE}.$t
# just to be sure flush all disk operations before uploading
flush
curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
}
try=1
while true; do
echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
export TEST_YML_PATH
/bin/bash $DEPLOYMENT_SCRIPT >> $LOG_FILE 2>&1
result=$?
if [ $result == 0 ] ; then
echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
break
fi
if (( try > TRIES )) ; then
echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
break
fi
# wait WAITTIME before checking the state
echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
sleep $WAITTIME
((try++))
done
exit $result
instance-deploy-test.cfg.jinja2 0000664 0000000 0000000 00000003562 13712426147 0037775 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test {# Get test type, default to script-from-url, as defined in instance-input-schema.json -#}
{% set test_type = slapparameter_dict.get('test-type', 'script-from-url') -%}
{# Choose parameters according to test type -#}
{% if test_type == 'script-from-url' -%}
{% set script_url = slapparameter_dict.get('script-to-test-url') -%}
{% set test_yml_path = '/not/required' -%}
{% elif test_type == 'cloned-playbook' -%}
{% set script_url = 'http://10.0.2.100/standalone-local-playbook' -%}
{% set test_yml_path = slapparameter_dict.get('yml-path-to-test') -%}
{% else -%}
The test_type = "{{ test_type }}" is unsupported.
{% endif -%}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
request-kvm
[request-kvm]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm
name = test-kvm
sla-computer_guid = ${slap-connection:computer-id}
# Tested image
# Passed by request
config-virtual-hard-drive-url = {{ slapparameter_dict.get('image-to-test-url') }}
config-virtual-hard-drive-md5sum = {{ slapparameter_dict.get('image-to-test-md5sum') }}
# The test script
config-bootstrap-script-url = {{ in_vm_test_script }}#{{ in_vm_test_script_md5 }}
# Script configuration
config-data-to-vm =
URL={{ script_url }}
WAITTIME={{ waittime }}
TRIES={{ tries }}
TEST_YML_PATH={{ test_yml_path }}
# require HTTP server
config-enable-http-server = true
# VM options
config-ram-size = 4096
config-cpu-count = 2
# await for system to be ready
return =
url
backend-url
ipv6
nat-rule-port-443
nat-rule-port-80
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc/
var = ${:home}/var/
srv = ${:home}/srv/
bin = ${:home}/bin/
tmp = ${:home}/tmp/
log = ${:var}/log/
services = ${:etc}/service/
scripts = ${:etc}/run/
instance-input-schema.json 0000664 0000000 0000000 00000003234 13712426147 0037153 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test {
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"image-to-test-url": {
"title": "Image To Test (URL)",
"description": "Absolute URL to QCOW2 vm-bootstrap compatible image.",
"type": "string"
},
"image-to-test-md5sum": {
"title": "MD5 checksum of Image To Test.",
"description": "MD5 checksum of QCOW2 vm-bootstrap compatible image.",
"type": "string"
},
"test-relative-directory": {
"title": "Relative directory where tests are found.",
"description": "The base of this directory is [slapos-package].",
"type": "string"
},
"test-type": {
"title": "Type of the test.",
"type": "string",
"default": "script-from-url",
"enum": [
"script-from-url",
"cloned-playbook"
]
},
"script-to-test-url": {
"title": "Optional URL of script to test, used for test-type=script-from-url.",
"description": "Optional URL of script to test, which will be injected into the VM.\nThis parameter will NOT use local git clone of [slapos-package] part, it will test fully provided script.",
"type": "string"
},
"yml-path-to-test": {
"title": "Optional YML path to test from the playbook, used for test-type=cloned-playbook.",
"description": "Optional YML path to test from the playbook, provided as relative path to playbook root.\nThis parameter will use local git clone of [slapos-package] part, from which playbook will be constructed.",
"type": "string"
}
},
"required": [
"image-to-test-url",
"image-to-test-md5sum",
"test-type"
]
}
instance.cfg.in 0000664 0000000 0000000 00000004727 13712426147 0034763 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test [buildout]
extends = ${template:output}
parts +=
runTestSuite
copy-test-data
[copy-test-data]
# Note: This is a hack, as such while requesting KVM it is not possible
# to fill correctly http directory
# Note: This will only work inside of non-user related environment
# dest shell variable will evaluate to requested partition's
# srv/public directory
# The "if" statement makes this script be run only in partition
# *without* srv/public - so according to current knowledge, the
# one which requests KVM backend for testing
recipe = plone.recipe.command
stop-on-error = true
update-command = $${:command}
command =
if [ ! -d srv/public ] ; then
dest=`echo ../*/srv/public`
cp ${playbook:output} $dest/playbook.tar.gz &&
cp ${standalone-local-playbook:location} $dest/standalone-local-playbook
fi
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
template = inline:
#!/bin/sh
export PATH=${python-with-eggs:location}:$PATH
exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory', 'playbook/roles/standalone-shared') }}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
mode = 0755
context =
key slapparameter_dict slap-configuration:configuration
key partition_ipv4 slap-configuration:ipv4
[switch_softwaretype]
default = $${:deploy-test}
deploy-test = $${dynamic-template-deploy-test:rendered}
[dynamic-template-deploy-test]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
rendered = $${buildout:directory}/template-deploy-test.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
raw in_vm_test_script ${deploy-script-controller-script:location}
raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
raw waittime ${deploy-script-controller-script:waittime}
raw tries ${deploy-script-controller-script:tries}
mode = 0644
runTestSuite.py 0000664 0000000 0000000 00000014633 13712426147 0035116 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test from __future__ import print_function
import argparse
import os
import glob
from time import gmtime, strftime, time, sleep
from erp5.util import taskdistribution, testsuite
import logging
import sys
import tempfile
import json
SLEEP_TIME = 15
TRY_AMOUNT = 3600
def waitForSite(partition_path):
status_dict = {'command': 'file not found'}
# find test result, wait 10h
try_num = 1
start = time()
result_found = False
while 1:
finished = False
try_info = 'Try %s/%s: ' % (try_num, TRY_AMOUNT)
test_result_glob = os.path.join(
partition_path,
'..',
'*',
'srv',
'public',
'test-script-result',
)
print(try_info + 'Waiting for data in %r.' % (test_result_glob,))
result_list = glob.glob(test_result_glob)
if len(result_list) > 0:
result_path = result_list[0]
print(try_info + 'Data directory %r found, looking for results.' % (
result_path,))
result_file_list = list((
os.path.join(dirname, filename)
for dirname, dirnames, filenames in os.walk(result_path)
for filename in filenames
))
if len(result_file_list):
print(try_info + 'No result posted, will check next try.')
for result_file in result_file_list:
print(try_info + 'Data found.')
result_found = True
result_file = os.path.abspath(result_file)
status_dict['command'] = result_file
result = open(result_file).read()
# remove result, as it is not required anymore
os.unlink(result_file)
print(try_info + 'Analysis of result %r:' % (result_file,))
print(try_info + result)
status_dict['stderr'] = 'Last result:\n%s' % (result,)
if 'FATAL: all hosts have already failed -- aborting' in result:
# failed
status_dict.update(
success=False
)
finished = False
status_dict['stdout'] = try_info + 'Build not yet successful.'
print(try_info + '%r: Found not yet finished run.' % (result_file,))
elif "Build successful, connect to:" in result:
# success
status_dict.update(
success=True
)
finished = True
print(try_info + '%r: Found finished successful run.' % (
result_file,))
status_dict['stdout'] = try_info + 'Build successful.'
break
else:
# unknown
status_dict.update(
success=False
)
status_dict['stdout'] = \
try_info + 'Cannot find success nor failure result in the output'
print(try_info + '%r: Found unknown run.' % (result_file,))
finished = False
if finished:
break
if try_num >= TRY_AMOUNT:
msg = try_info + 'Time exceeded, success not found.'
print(msg)
status_dict.setdefault('stdout', '')
status_dict['stdout'] = '\n'.join([status_dict['stdout'], msg])
break
try_num += 1
print(try_info + 'Sleeping for %ss.' % (SLEEP_TIME,))
sleep(SLEEP_TIME)
if not result_found:
status_dict['stdout'] = try_info + 'Test timed out and no result found.'
status_dict.update(
success=False
)
end = time()
status_dict.update(
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start,
)
print(try_info + 'status_dict %r' % (status_dict,))
return status_dict
def main():
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.DEBUG)
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', type=int,
help='Number of CPUs to use for the VM')
parser.add_argument('--master_url',
help='The Url of Master controlling test suites')
# SlapOS and deploy test specific
parser.add_argument(
'--partition_path',
help="Path of a partition",
default=os.path.abspath(os.getcwd()))
parser.add_argument(
'--test_reference',
help="Reference of the test",
default="missing"
)
parser.add_argument(
'--partition_ipv4',
help="IPv4 of a partition"
)
parser.add_argument(
'--test_location',
help="Location of the tests"
)
parser.add_argument(
'--python_interpreter',
help="Path to python interpreter used to run the test suite"
)
args = parser.parse_args()
revision = args.revision
test_suite_title = args.test_suite_title or args.test_suite
suite = testsuite.EggTestSuite(
1, test_suite=args.test_suite, node_quantity=args.node_quantity,
python_interpreter=args.python_interpreter,
egg_test_path_dict={
os.path.basename(os.path.normpath(path)): path
for path in args.test_location.split(',')},
revision=revision)
access_url_http = None
access_url_https = None
if args.partition_ipv4:
access_url_http = 'http://%s:10080' % (args.partition_ipv4,)
access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
distributor = taskdistribution.TaskDistributor(
args.master_url,
logger=logger)
test_result = distributor.createTestResult(
revision, suite.getTestList(), args.test_node_title,
suite.allow_restart, test_suite_title, args.project_title)
if test_result is None:
return
# Create the site
status_dict = waitForSite(args.partition_path)
status_file = tempfile.NamedTemporaryFile()
status_file.write(json.dumps(status_dict))
status_file.flush()
os.fsync(status_file.fileno())
os.environ['TEST_SITE_STATUS_JSON'] = status_file.name
assert revision == test_result.revision, (revision, test_result.revision)
while suite.acquire():
test = test_result.start(suite.running.keys())
if test is not None:
suite.start(test.name, lambda status_dict,
__test=test: __test.stop(**status_dict))
elif not suite.running:
break
return
if __name__ == "__main__":
main()
software.cfg 0000664 0000000 0000000 00000003357 13712426147 0034402 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test [buildout]
extends =
buildout.hash.cfg
https://lab.nexedi.com/nexedi/slapos/raw/1.0.134/software/kvm/software.cfg
parts =
eggs
template-deploy-test
runTestSuite_py
playbook
[runTestSuite_py]
recipe = zc.recipe.egg
eggs =
erp5.util
interpreter = ${:_buildout_section_name_}
[python-with-eggs]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python &&
ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python2.7
update-command = ${:command}
[playbook]
recipe = plone.recipe.command
stop-on-error = true
environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
location = ${buildout:parts-directory}/${:_buildout_section_name_}
output = ${:location}/playbook.tar.gz
command =
${:environment}
rm -fr ${:location}
mkdir -p ${:location}
cd ${slapos-package:location}/playbook
tar czf ${:output} .
update-command = ${:command}
[test-location]
base = ${slapos-package:location}
[slapos-package]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.package.git
branch = cdn-test
git-executable = ${git:location}/bin/git
[template]
output = ${buildout:directory}/template-original.kvm.cfg
[deploy-script-controller-script]
location = ${:_profile_base_location_}/${:filename}
# configuration
waittime = 360
tries = 80
[standalone-local-playbook]
filename = standalone-local-playbook
location = ${:_profile_base_location_}/${:filename}
[template-deploy-test]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
software.cfg.json 0000664 0000000 0000000 00000000452 13712426147 0035343 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test {
"name": "Deploy Test",
"description": "Deploy Testing software release",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standard ERP5TestNode hooked",
"request": "instance-input-schema.json",
"index": 0
}
}
}
standalone-local-playbook 0000664 0000000 0000000 00000206432 13712426147 0037047 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/deploy-test #!/bin/bash
set -e
#
# Paste content of function-common
# https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/roles/install-script/files/function-common
#
#!/bin/bash
#
# functions-common - Common functions used by DevStack components
#
# The canonical copy of this file is maintained in the DevStack repo.
# All modifications should be made there and then sync'ed to other repos
# as required.
#
# This file is sorted alphabetically within the function groups.
#
# - Config Functions
# - Control Functions
# - Distro Functions
# - Git Functions
# - OpenStack Functions
# - Package Functions
# - Process Functions
# - Service Functions
# - System Functions
#
# The following variables are assumed to be defined by certain functions:
#
# - ``ENABLED_SERVICES``
# - ``ERROR_ON_CLONE``
# - ``FILES``
# - ``OFFLINE``
# - ``RECLONE``
# - ``REQUIREMENTS_DIR``
# - ``STACK_USER``
# - ``TRACK_DEPENDS``
# - ``UNDO_REQUIREMENTS``
# - ``http_proxy``, ``https_proxy``, ``no_proxy``
#
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Global Config Variables
declare -A GITREPO
declare -A GITBRANCH
declare -A GITDIR
TRACK_DEPENDS=${TRACK_DEPENDS:-False}
# Normalize config values to True or False
# Accepts as False: 0 no No NO false False FALSE
# Accepts as True: 1 yes Yes YES true True TRUE
# VAR=$(trueorfalse default-value test-value)
function trueorfalse {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local default=$1
local literal=$2
local testval=${!literal:-}
[[ -z "$testval" ]] && { echo "$default"; return; }
[[ "0 no No NO false False FALSE" =~ "$testval" ]] && { echo "False"; return; }
[[ "1 yes Yes YES true True TRUE" =~ "$testval" ]] && { echo "True"; return; }
echo "$default"
$xtrace
}
#function isset {
# [[ -v "$1" ]]
#}
#
# Control Functions
# =================
# Prints backtrace info
# filename:lineno:function
# backtrace level
function backtrace {
local level=$1
local deep=$((${#BASH_SOURCE[@]} - 1))
echo "[Call Trace]"
while [ $level -le $deep ]; do
echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
deep=$((deep - 1))
done
}
# Prints line number and "message" then exits
# die $LINENO "message"
function die {
local exitcode=$?
set +o xtrace
local line=$1; shift
if [ $exitcode == 0 ]; then
exitcode=1
fi
backtrace 2
err $line "$*"
# Give buffers a second to flush
sleep 1
exit $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message" and exits
# NOTE: env-var is the variable name without a '$'
# die_if_not_set $LINENO env-var "message"
function die_if_not_set {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
die $line "$*"
fi
$xtrace
}
# Prints line number and "message" in error format
# err $LINENO "message"
function err {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
echo $msg 1>&2;
if [[ -n ${LOGDIR} ]]; then
echo $msg >> "${LOGDIR}/error.log"
fi
$xtrace
return $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message"
# NOTE: env-var is the variable name without a '$'
# err_if_not_set $LINENO env-var "message"
function err_if_not_set {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
err $line "$*"
fi
$xtrace
return $exitcode
}
# Exit after outputting a message about the distribution not being supported.
# exit_distro_not_supported [optional-string-telling-what-is-missing]
function exit_distro_not_supported {
if [[ -z "$DISTRO" ]]; then
GetDistro
fi
if [ $# -gt 0 ]; then
die $LINENO "Support for $DISTRO is incomplete: no support for $@"
else
die $LINENO "Support for $DISTRO is incomplete."
fi
}
# Test if the named environment variable is set and not zero length
# is_set env-var
function is_set {
local var=\$"$1"
eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this
}
# Prints line number and "message" in warning format
# warn $LINENO "message"
function warn {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
echo $msg 1>&2;
if [[ -n ${LOGDIR} ]]; then
echo $msg >> "${LOGDIR}/error.log"
fi
$xtrace
return $exitcode
}
# Distro Functions
# ================
# Determine OS Vendor, Release and Update
# Tested with OS/X, Ubuntu, RedHat, CentOS, Fedora
# Returns results in global variables:
# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
# ``os_RELEASE`` - major release: ``14.04`` (Ubuntu), ``20`` (Fedora)
# ``os_UPDATE`` - update: ex. the ``5`` in ``RHEL6.5``
# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
# ``os_CODENAME`` - vendor's codename for release: ``snow leopard``, ``trusty``
os_VENDOR=""
os_RELEASE=""
os_UPDATE=""
os_PACKAGE=""
os_CODENAME=""
# GetOSVersion
function GetOSVersion {
# Figure out which vendor we are
if [[ -x "`which sw_vers 2>/dev/null`" ]]; then
# OS/X
os_VENDOR=`sw_vers -productName`
os_RELEASE=`sw_vers -productVersion`
os_UPDATE=${os_RELEASE##*.}
os_RELEASE=${os_RELEASE%.*}
os_PACKAGE=""
if [[ "$os_RELEASE" =~ "10.7" ]]; then
os_CODENAME="lion"
elif [[ "$os_RELEASE" =~ "10.6" ]]; then
os_CODENAME="snow leopard"
elif [[ "$os_RELEASE" =~ "10.5" ]]; then
os_CODENAME="leopard"
elif [[ "$os_RELEASE" =~ "10.4" ]]; then
os_CODENAME="tiger"
elif [[ "$os_RELEASE" =~ "10.3" ]]; then
os_CODENAME="panther"
else
os_CODENAME=""
fi
elif [[ -x $(which lsb_release 2>/dev/null) ]]; then
os_VENDOR=$(lsb_release -i -s)
os_RELEASE=$(lsb_release -r -s)
os_UPDATE=""
os_PACKAGE="rpm"
if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
os_PACKAGE="deb"
elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
lsb_release -d -s | grep -q openSUSE
if [[ $? -eq 0 ]]; then
os_VENDOR="openSUSE"
fi
elif [[ $os_VENDOR == "openSUSE project" ]]; then
os_VENDOR="openSUSE"
elif [[ $os_VENDOR =~ Red.*Hat ]]; then
os_VENDOR="Red Hat"
fi
os_CODENAME=$(lsb_release -c -s)
elif [[ -r /etc/redhat-release ]]; then
# Red Hat Enterprise Linux Server release 5.5 (Tikanga)
# Red Hat Enterprise Linux Server release 7.0 Beta (Maipo)
# CentOS release 5.5 (Final)
# CentOS Linux release 6.0 (Final)
# Fedora release 16 (Verne)
# XenServer release 6.2.0-70446c (xenenterprise)
# Oracle Linux release 7
os_CODENAME=""
for r in "Red Hat" CentOS Fedora XenServer; do
os_VENDOR=$r
if [[ -n "`grep \"$r\" /etc/redhat-release`" ]]; then
ver=`sed -e 's/^.* \([0-9].*\) (\(.*\)).*$/\1\|\2/' /etc/redhat-release`
os_CODENAME=${ver#*|}
os_RELEASE=${ver%|*}
os_UPDATE=${os_RELEASE##*.}
os_RELEASE=${os_RELEASE%.*}
break
fi
os_VENDOR=""
done
if [ "$os_VENDOR" = "Red Hat" ] && [[ -r /etc/oracle-release ]]; then
os_VENDOR=OracleLinux
fi
os_PACKAGE="rpm"
elif [[ -r /etc/SuSE-release ]]; then
for r in openSUSE "SUSE Linux"; do
if [[ "$r" = "SUSE Linux" ]]; then
os_VENDOR="SUSE LINUX"
else
os_VENDOR=$r
fi
if [[ -n "`grep \"$r\" /etc/SuSE-release`" ]]; then
os_CODENAME=`grep "CODENAME = " /etc/SuSE-release | sed 's:.* = ::g'`
os_RELEASE=`grep "VERSION = " /etc/SuSE-release | sed 's:.* = ::g'`
os_UPDATE=`grep "PATCHLEVEL = " /etc/SuSE-release | sed 's:.* = ::g'`
break
fi
os_VENDOR=""
done
os_PACKAGE="rpm"
# If lsb_release is not installed, we should be able to detect Debian OS
elif [[ -f /etc/debian_version ]] && [[ $(cat /proc/version) =~ "Debian" ]]; then
os_VENDOR="Debian"
os_PACKAGE="deb"
os_CODENAME=$(awk '/VERSION=/' /etc/os-release | sed 's/VERSION=//' | sed -r 's/\"|\(|\)//g' | awk '{print $2}')
os_RELEASE=$(awk '/VERSION_ID=/' /etc/os-release | sed 's/VERSION_ID=//' | sed 's/\"//g')
fi
export os_VENDOR os_RELEASE os_UPDATE os_PACKAGE os_CODENAME
}
# Translate the OS version values into common nomenclature
# Sets global ``DISTRO`` from the ``os_*`` values
declare DISTRO
function GetDistro {
GetOSVersion
if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
# 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
DISTRO=$os_CODENAME
elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
# For Fedora, just use 'f' and the release
DISTRO="f$os_RELEASE"
elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
DISTRO="opensuse-$os_RELEASE"
elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
# For SLE, also use the service pack
if [[ -z "$os_UPDATE" ]]; then
DISTRO="sle${os_RELEASE}"
else
DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
fi
elif [[ "$os_VENDOR" =~ (Red Hat) || \
"$os_VENDOR" =~ (CentOS) || \
"$os_VENDOR" =~ (OracleLinux) ]]; then
# Drop the . release as we assume it's compatible
DISTRO="rhel${os_RELEASE::1}"
elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
DISTRO="xs$os_RELEASE"
else
# Catch-all for now is Vendor + Release + Update
DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
fi
export DISTRO
}
# Utility function for checking machine architecture
# is_arch arch-type
function is_arch {
[[ "$(uname -m)" == "$1" ]]
}
# Determine if current distribution is an Oracle distribution
# is_oraclelinux
function is_oraclelinux {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "OracleLinux" ]
}
# Determine if current distribution is a Fedora-based distribution
# (Fedora, RHEL, CentOS, etc).
# is_fedora
function is_fedora {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
[ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleLinux" ]
}
# Determine if current distribution is a SUSE-based distribution
# (openSUSE, SLE).
# is_suse
function is_suse {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "openSUSE" ] || [ "$os_VENDOR" = "SUSE LINUX" ]
}
# Determine if current distribution is an Ubuntu-based distribution
# It will also detect non-Ubuntu but Debian-based distros
# is_ubuntu
function is_ubuntu {
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
[ "$os_PACKAGE" = "deb" ]
}
# Git Functions
# =============
# Returns openstack release name for a given branch name
# ``get_release_name_from_branch branch-name``
function get_release_name_from_branch {
local branch=$1
if [[ $branch =~ "stable/" || $branch =~ "proposed/" ]]; then
echo ${branch#*/}
else
echo "master"
fi
}
# git clone only if directory doesn't exist already. Since ``DEST`` might not
# be owned by the installation user, we create the directory and change the
# ownership to the proper user.
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
# does not exist (default is False, meaning the repo will be cloned).
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
# git_clone remote dest-dir branch
function git_clone {
local git_remote=$1
local git_dest=$2
local git_ref=$3
local orig_dir=$(pwd)
local git_clone_flags=""
RECLONE=$(trueorfalse False RECLONE)
if [[ "${GIT_DEPTH}" -gt 0 ]]; then
git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
fi
if [[ "$OFFLINE" = "True" ]]; then
echo "Running in offline mode, clones already exist"
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
return
fi
if echo $git_ref | egrep -q "^refs"; then
# If our branch name is a gerrit style refs/changes/...
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
fi
cd $git_dest
git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
else
# do a full clone only if the directory doesn't exist
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
cd $git_dest
# This checkout syntax works for both branches and tags
git checkout $git_ref
elif [[ "$RECLONE" = "True" ]]; then
# if it does exist then simulate what clone does if asked to RECLONE
cd $git_dest
# set the url to pull from and fetch
git remote set-url origin $git_remote
git_timed fetch origin
# remove the existing ignored files (like pyc) as they cause breakage
# (due to the py files having older timestamps than our pyc, so python
# thinks the pyc files are correct using them)
find $git_dest -name '*.pyc' -delete
# handle git_ref accordingly to type (tag, branch)
if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
git_update_tag $git_ref
elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
git_update_branch $git_ref
elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
git_update_remote_branch $git_ref
else
die $LINENO "$git_ref is neither branch nor tag"
fi
fi
fi
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
}
# A variation on git clone that lets us specify a project by it's
# actual name, like oslo.config. This is exceptionally useful in the
# library installation case
function git_clone_by_name {
local name=$1
local repo=${GITREPO[$name]}
local dir=${GITDIR[$name]}
local branch=${GITBRANCH[$name]}
git_clone $repo $dir $branch
}
# git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end. This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang. GIT_TIMEOUT, if set, is passed directly
# to timeout(1); otherwise the default value of 0 maintains the status
# quo of waiting forever.
# usage: git_timed
function git_timed {
local count=0
local timeout=0
if [[ -n "${GIT_TIMEOUT}" ]]; then
timeout=${GIT_TIMEOUT}
fi
until timeout -s SIGINT ${timeout} git "$@"; do
# 124 is timeout(1)'s special return code when it reached the
# timeout; otherwise assume fatal failure
if [[ $? -ne 124 ]]; then
die $LINENO "git call failed: [git $@]"
fi
count=$(($count + 1))
warn "timeout ${count} for git call: [git $@]"
if [ $count -eq 3 ]; then
die $LINENO "Maximum of 3 git retries reached"
fi
sleep 5
done
}
# git update using reference as a branch.
# git_update_branch ref
function git_update_branch {
local git_branch=$1
git checkout -f origin/$git_branch
# a local branch might not exist
git branch -D $git_branch || true
git checkout -b $git_branch
}
# git update using reference as a branch.
# git_update_remote_branch ref
function git_update_remote_branch {
local git_branch=$1
git checkout -b $git_branch -t origin/$git_branch
}
# git update using reference as a tag. Be careful editing source at that repo
# as working copy will be in a detached mode
# git_update_tag ref
function git_update_tag {
local git_tag=$1
git tag -d $git_tag
# fetching given tag only
git_timed fetch origin tag $git_tag
git checkout -f $git_tag
}
# OpenStack Functions
# ===================
# Get the default value for HOST_IP
# get_default_host_ip fixed_range floating_range host_ip_iface host_ip
function get_default_host_ip {
local fixed_range=$1
local floating_range=$2
local host_ip_iface=$3
local host_ip=$4
# Search for an IP unless an explicit is set by ``HOST_IP`` environment variable
if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
host_ip=""
# Find the interface used for the default route
host_ip_iface=${host_ip_iface:-$(ip route | awk '/default/ {print $5}' | head -1)}
local host_ips=$(LC_ALL=C ip -f inet addr show ${host_ip_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}')
local ip
for ip in $host_ips; do
# Attempt to filter out IP addresses that are part of the fixed and
# floating range. Note that this method only works if the ``netaddr``
# python library is installed. If it is not installed, an error
# will be printed and the first IP from the interface will be used.
# If that is not correct set ``HOST_IP`` in ``localrc`` to the correct
# address.
if ! (address_in_net $ip $fixed_range || address_in_net $ip $floating_range); then
host_ip=$ip
break;
fi
done
fi
echo $host_ip
}
# Generates hex string from ``size`` byte of pseudo random data
# generate_hex_string size
function generate_hex_string {
local size=$1
hexdump -n "$size" -v -e '/1 "%02x"' /dev/urandom
}
# Grab a numbered field from python prettytable output
# Fields are numbered starting with 1
# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
# get_field field-number
function get_field {
local data field
while read data; do
if [ "$1" -lt 0 ]; then
field="(\$(NF$1))"
else
field="\$$(($1 + 1))"
fi
echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
done
}
# install default policy
# copy over a default policy.json and policy.d for projects
function install_default_policy {
local project=$1
local project_uc=$(echo $1|tr a-z A-Z)
local conf_dir="${project_uc}_CONF_DIR"
# eval conf dir to get the variable
conf_dir="${!conf_dir}"
local project_dir="${project_uc}_DIR"
# eval project dir to get the variable
project_dir="${!project_dir}"
local sample_conf_dir="${project_dir}/etc/${project}"
local sample_policy_dir="${project_dir}/etc/${project}/policy.d"
# first copy any policy.json
cp -p $sample_conf_dir/policy.json $conf_dir
# then optionally copy over policy.d
if [[ -d $sample_policy_dir ]]; then
cp -r $sample_policy_dir $conf_dir/policy.d
fi
}
# Add a policy to a policy.json file
# Do nothing if the policy already exists
# ``policy_add policy_file policy_name policy_permissions``
function policy_add {
local policy_file=$1
local policy_name=$2
local policy_perm=$3
if grep -q ${policy_name} ${policy_file}; then
echo "Policy ${policy_name} already exists in ${policy_file}"
return
fi
# Add a terminating comma to policy lines without one
# Remove the closing '}' and all lines following to the end-of-file
local tmpfile=$(mktemp)
uniq ${policy_file} | sed -e '
s/]$/],/
/^[}]/,$d
' > ${tmpfile}
# Append policy and closing brace
echo " \"${policy_name}\": ${policy_perm}" >>${tmpfile}
echo "}" >>${tmpfile}
mv ${tmpfile} ${policy_file}
}
# Gets or creates a domain
# Usage: get_or_create_domain
function get_or_create_domain {
local os_url="$KEYSTONE_SERVICE_URI_V3"
# Gets domain id
local domain_id=$(
# Gets domain id
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 domain show $1 \
-f value -c id 2>/dev/null ||
# Creates new domain
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 domain create $1 \
--description "$2" \
-f value -c id
)
echo $domain_id
}
# Gets or creates group
# Usage: get_or_create_group [ ]
function get_or_create_group {
local domain=${2:+--domain ${2}}
local desc="${3:-}"
local os_url="$KEYSTONE_SERVICE_URI_V3"
# Gets group id
local group_id=$(
# Creates new group with --or-show
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 group create $1 \
$domain --description "$desc" --or-show \
-f value -c id
)
echo $group_id
}
# Gets or creates user
# Usage: get_or_create_user [ []]
function get_or_create_user {
if [[ ! -z "$3" ]]; then
local email="--email=$3"
else
local email=""
fi
local os_cmd="openstack"
local domain=""
if [[ ! -z "$4" ]]; then
domain="--domain=$4"
os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
fi
# Gets user id
local user_id=$(
# Creates new user with --or-show
$os_cmd user create \
$1 \
--password "$2" \
$email \
$domain \
--or-show \
-f value -c id
)
echo $user_id
}
# Gets or creates project
# Usage: get_or_create_project []
function get_or_create_project {
# Gets project id
local os_cmd="openstack"
local domain=""
if [[ ! -z "$2" ]]; then
domain="--domain=$2"
os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
fi
local project_id=$(
# Creates new project with --or-show
$os_cmd project create $1 $domain --or-show -f value -c id
)
echo $project_id
}
# Gets or creates role
# Usage: get_or_create_role
function get_or_create_role {
local role_id=$(
# Creates role with --or-show
openstack role create $1 --or-show -f value -c id
)
echo $role_id
}
# Gets or adds user role to project
# Usage: get_or_add_user_project_role
function get_or_add_user_project_role {
# Gets user role id
local user_role_id=$(openstack role list \
--user $2 \
--project $3 \
--column "ID" \
--column "Name" \
| grep " $1 " | get_field 1)
if [[ -z "$user_role_id" ]]; then
# Adds role to user
user_role_id=$(openstack role add \
$1 \
--user $2 \
--project $3 \
| grep " id " | get_field 2)
fi
echo $user_role_id
}
# Gets or adds group role to project
# Usage: get_or_add_group_project_role
function get_or_add_group_project_role {
# Gets group role id
local group_role_id=$(openstack role list \
--group $2 \
--project $3 \
--column "ID" \
--column "Name" \
| grep " $1 " | get_field 1)
if [[ -z "$group_role_id" ]]; then
# Adds role to group
group_role_id=$(openstack role add \
$1 \
--group $2 \
--project $3 \
| grep " id " | get_field 2)
fi
echo $group_role_id
}
# Gets or creates service
# Usage: get_or_create_service
function get_or_create_service {
# Gets service id
local service_id=$(
# Gets service id
openstack service show $1 -f value -c id 2>/dev/null ||
# Creates new service if not exists
openstack service create \
$2 \
--name $1 \
--description="$3" \
-f value -c id
)
echo $service_id
}
# Gets or creates endpoint
# Usage: get_or_create_endpoint
function get_or_create_endpoint {
# Gets endpoint id
local endpoint_id=$(openstack endpoint list \
--column "ID" \
--column "Region" \
--column "Service Name" \
| grep " $2 " \
| grep " $1 " | get_field 1)
if [[ -z "$endpoint_id" ]]; then
# Creates new endpoint
endpoint_id=$(openstack endpoint create \
$1 \
--region $2 \
--publicurl $3 \
--adminurl $4 \
--internalurl $5 \
| grep " id " | get_field 2)
fi
echo $endpoint_id
}
# Package Functions
# =================
# _get_package_dir
function _get_package_dir {
local base_dir=$1
local pkg_dir
if [[ -z "$base_dir" ]]; then
base_dir=$FILES
fi
if is_ubuntu; then
pkg_dir=$base_dir/debs
elif is_fedora; then
pkg_dir=$base_dir/rpms
elif is_suse; then
pkg_dir=$base_dir/rpms-suse
else
exit_distro_not_supported "list of packages"
fi
echo "$pkg_dir"
}
# Wrapper for ``apt-get`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``
# apt_get operation package [package ...]
function apt_get {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
[[ "$OFFLINE" = "True" || -z "$@" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
$xtrace
$sudo DEBIAN_FRONTEND=noninteractive \
http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
no_proxy=${no_proxy:-} \
apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@"
}
function _parse_package_files {
local files_to_parse=$@
if [[ -z "$DISTRO" ]]; then
GetDistro
fi
for fname in ${files_to_parse}; do
local OIFS line package distros distro
[[ -e $fname ]] || continue
OIFS=$IFS
IFS=$'\n'
for line in $(<${fname}); do
if [[ $line =~ "NOPRIME" ]]; then
continue
fi
# Assume we want this package
package=${line%#*}
inst_pkg=1
# Look for # dist:xxx in comment
if [[ $line =~ (.*)#.*dist:([^ ]*) ]]; then
# We are using BASH regexp matching feature.
package=${BASH_REMATCH[1]}
distros=${BASH_REMATCH[2]}
# In bash ${VAR,,} will lowecase VAR
# Look for a match in the distro list
if [[ ! ${distros,,} =~ ${DISTRO,,} ]]; then
# If no match then skip this package
inst_pkg=0
fi
fi
if [[ $inst_pkg = 1 ]]; then
echo $package
fi
done
IFS=$OIFS
done
}
# get_packages() collects a list of package names of any type from the
# prerequisite files in ``files/{debs|rpms}``. The list is intended
# to be passed to a package installer such as apt or yum.
#
# Only packages required for the services in 1st argument will be
# included. Two bits of metadata are recognized in the prerequisite files:
#
# - ``# NOPRIME`` defers installation to be performed later in `stack.sh`
# - ``# dist:DISTRO`` or ``dist:DISTRO1,DISTRO2`` limits the selection
# of the package to the distros listed. The distro names are case insensitive.
function get_packages {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local services=$@
local package_dir=$(_get_package_dir)
local file_to_parse=""
local service=""
if [[ -z "$package_dir" ]]; then
echo "No package directory supplied"
return 1
fi
for service in ${services//,/ }; do
# Allow individual services to specify dependencies
if [[ -e ${package_dir}/${service} ]]; then
file_to_parse="${file_to_parse} ${package_dir}/${service}"
fi
# NOTE(sdague) n-api needs glance for now because that's where
# glance client is
if [[ $service == n-api ]]; then
if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
file_to_parse="${file_to_parse} ${package_dir}/nova"
fi
if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
file_to_parse="${file_to_parse} ${package_dir}/glance"
fi
elif [[ $service == c-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/cinder ]]; then
file_to_parse="${file_to_parse} ${package_dir}/cinder"
fi
elif [[ $service == ceilometer-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/ceilometer ]]; then
file_to_parse="${file_to_parse} ${package_dir}/ceilometer"
fi
elif [[ $service == s-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/swift ]]; then
file_to_parse="${file_to_parse} ${package_dir}/swift"
fi
elif [[ $service == n-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
file_to_parse="${file_to_parse} ${package_dir}/nova"
fi
elif [[ $service == g-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
file_to_parse="${file_to_parse} ${package_dir}/glance"
fi
elif [[ $service == key* ]]; then
if [[ ! $file_to_parse =~ $package_dir/keystone ]]; then
file_to_parse="${file_to_parse} ${package_dir}/keystone"
fi
elif [[ $service == q-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/neutron ]]; then
file_to_parse="${file_to_parse} ${package_dir}/neutron"
fi
elif [[ $service == ir-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/ironic ]]; then
file_to_parse="${file_to_parse} ${package_dir}/ironic"
fi
fi
done
echo "$(_parse_package_files $file_to_parse)"
$xtrace
}
# get_plugin_packages() collects a list of package names of any type from a
# plugin's prerequisite files in ``$PLUGIN/devstack/files/{debs|rpms}``. The
# list is intended to be passed to a package installer such as apt or yum.
#
# Only packages required for enabled and collected plugins will included.
#
# The same metadata used in the main DevStack prerequisite files may be used
# in these prerequisite files, see get_packages() for more info.
function get_plugin_packages {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local files_to_parse=""
local package_dir=""
for plugin in ${DEVSTACK_PLUGINS//,/ }; do
local package_dir="$(_get_package_dir ${GITDIR[$plugin]}/devstack/files)"
files_to_parse+="$package_dir/$plugin"
done
echo "$(_parse_package_files $files_to_parse)"
$xtrace
}
# Distro-agnostic package installer
# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
# install_package package [package ...]
function update_package_repo {
NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
REPOS_UPDATED=${REPOS_UPDATED:-False}
RETRY_UPDATE=${RETRY_UPDATE:-False}
if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
return 0
fi
if is_ubuntu; then
local xtrace=$(set +o | grep xtrace)
set +o xtrace
if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
# if there are transient errors pulling the updates, that's fine.
# It may be secondary repositories that we don't really care about.
apt_get update || /bin/true
REPOS_UPDATED=True
fi
$xtrace
fi
}
function real_install_package {
if is_ubuntu; then
apt_get install "$@"
elif is_fedora; then
yum_install "$@"
elif is_suse; then
zypper_install "$@"
else
exit_distro_not_supported "installing packages"
fi
}
# Distro-agnostic package installer
# install_package package [package ...]
function install_package {
update_package_repo
real_install_package $@ || RETRY_UPDATE=True update_package_repo && real_install_package $@
}
# Distro-agnostic function to tell if a package is installed
# is_package_installed package [package ...]
function is_package_installed {
if [[ -z "$@" ]]; then
return 1
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
if [[ "$os_PACKAGE" = "deb" ]]; then
dpkg -s "$@" > /dev/null 2> /dev/null
elif [[ "$os_PACKAGE" = "rpm" ]]; then
rpm --quiet -q "$@"
else
exit_distro_not_supported "finding if a package is installed"
fi
}
# Distro-agnostic package uninstaller
# uninstall_package package [package ...]
function uninstall_package {
if is_ubuntu; then
apt_get purge "$@"
elif is_fedora; then
sudo ${YUM:-yum} remove -y "$@" ||:
elif is_suse; then
sudo zypper rm "$@"
else
exit_distro_not_supported "uninstalling packages"
fi
}
# Wrapper for ``yum`` to set proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
# yum_install package [package ...]
function yum_install {
[[ "$OFFLINE" = "True" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
# The manual check for missing packages is because yum -y assumes
# missing packages are OK. See
# https://bugzilla.redhat.com/show_bug.cgi?id=965567
$sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
${YUM:-yum} install -y "$@" 2>&1 | \
awk '
BEGIN { fail=0 }
/No package/ { fail=1 }
{ print }
END { exit fail }' || \
die $LINENO "Missing packages detected"
# also ensure we catch a yum failure
if [[ ${PIPESTATUS[0]} != 0 ]]; then
die $LINENO "${YUM:-yum} install failure"
fi
}
# zypper wrapper to set arguments correctly
# Uses globals ``OFFLINE``, ``*_proxy``
# zypper_install package [package ...]
function zypper_install {
[[ "$OFFLINE" = "True" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
$sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
zypper --non-interactive install --auto-agree-with-licenses "$@"
}
# Process Functions
# =================
# _run_process() is designed to be backgrounded by run_process() to simulate a
# fork. It includes the dirty work of closing extra filehandles and preparing log
# files to produce the same logs as screen_it(). The log filename is derived
# from the service name.
# Uses globals ``CURRENT_LOG_TIME``, ``LOGDIR``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
# If an optional group is provided sg will be used to set the group of
# the command.
# _run_process service "command-line" [group]
function _run_process {
local service=$1
local command="$2"
local group=$3
# Undo logging redirections and close the extra descriptors
exec 1>&3
exec 2>&3
exec 3>&-
exec 6>&-
local real_logfile="${LOGDIR}/${service}.log.${CURRENT_LOG_TIME}"
if [[ -n ${LOGDIR} ]]; then
exec 1>&"$real_logfile" 2>&1
ln -sf "$real_logfile" ${LOGDIR}/${service}.log
if [[ -n ${SCREEN_LOGDIR} ]]; then
# Drop the backward-compat symlink
ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${service}.log
fi
# TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
export PYTHONUNBUFFERED=1
fi
# Run under ``setsid`` to force the process to become a session and group leader.
# The pid saved can be used with pkill -g to get the entire process group.
if [[ -n "$group" ]]; then
setsid sg $group "$command" & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
else
setsid $command & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
fi
# Just silently exit this process
exit 0
}
# Helper to remove the ``*.failure`` files under ``$SERVICE_DIR/$SCREEN_NAME``.
# This is used for ``service_check`` when all the ``screen_it`` are called finished
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
# init_service_check
function init_service_check {
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
mkdir -p "$SERVICE_DIR/$SCREEN_NAME"
fi
rm -f "$SERVICE_DIR/$SCREEN_NAME"/*.failure
}
# Find out if a process exists by partial name.
# is_running name
function is_running {
local name=$1
ps auxw | grep -v grep | grep ${name} > /dev/null
local exitcode=$?
# some times I really hate bash reverse binary logic
return $exitcode
}
# Run a single service under screen or directly
# If the command includes shell metachatacters (;<>*) it must be run using a shell
# If an optional group is provided sg will be used to run the
# command as that group.
# run_process service "command-line" [group]
function run_process {
local service=$1
local command="$2"
local group=$3
if is_service_enabled $service; then
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$service" "$command" "$group"
else
# Spawn directly without screen
_run_process "$service" "$command" "$group" &
fi
fi
}
# Helper to launch a process in a named screen
# Uses globals ``CURRENT_LOG_TIME``, ```LOGDIR``, ``SCREEN_LOGDIR``, `SCREEN_NAME``,
# ``SERVICE_DIR``, ``USE_SCREEN``
# screen_process name "command-line" [group]
# Run a command in a shell in a screen window, if an optional group
# is provided, use sg to set the group of the command.
function screen_process {
local name=$1
local command="$2"
local group=$3
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
screen -S $SCREEN_NAME -X screen -t $name
local real_logfile="${LOGDIR}/${name}.log.${CURRENT_LOG_TIME}"
echo "LOGDIR: $LOGDIR"
echo "SCREEN_LOGDIR: $SCREEN_LOGDIR"
echo "log: $real_logfile"
if [[ -n ${LOGDIR} ]]; then
screen -S $SCREEN_NAME -p $name -X logfile "$real_logfile"
screen -S $SCREEN_NAME -p $name -X log on
ln -sf "$real_logfile" ${LOGDIR}/${name}.log
if [[ -n ${SCREEN_LOGDIR} ]]; then
# Drop the backward-compat symlink
ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${1}.log
fi
fi
# sleep to allow bash to be ready to be send the command - we are
# creating a new window in screen and then sends characters, so if
# bash isn't running by the time we send the command, nothing
# happens. This sleep was added originally to handle gate runs
# where we needed this to be at least 3 seconds to pass
# consistently on slow clouds. Now this is configurable so that we
# can determine a reasonable value for the local case which should
# be much smaller.
sleep ${SCREEN_SLEEP:-3}
NL=`echo -ne '\015'`
# This fun command does the following:
# - the passed server command is backgrounded
# - the pid of the background process is saved in the usual place
# - the server process is brought back to the foreground
# - if the server process exits prematurely the fg command errors
# and a message is written to stdout and the process failure file
#
# The pid saved can be used in stop_process() as a process group
# id to kill off all child processes
if [[ -n "$group" ]]; then
command="sg $group '$command'"
fi
# Append the process to the screen rc file
screen_rc "$name" "$command"
screen -S $SCREEN_NAME -p $name -X stuff "$command & echo \$! >$SERVICE_DIR/$SCREEN_NAME/${name}.pid; fg || echo \"$name failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/${name}.failure\"$NL"
}
# Screen rc file builder
# Uses globals ``SCREEN_NAME``, ``SCREENRC``
# screen_rc service "command-line"
function screen_rc {
SCREEN_NAME=${SCREEN_NAME:-stack}
SCREENRC=$TOP_DIR/$SCREEN_NAME-screenrc
if [[ ! -e $SCREENRC ]]; then
# Name the screen session
echo "sessionname $SCREEN_NAME" > $SCREENRC
# Set a reasonable statusbar
echo "hardstatus alwayslastline '$SCREEN_HARDSTATUS'" >> $SCREENRC
# Some distributions override PROMPT_COMMAND for the screen terminal type - turn that off
echo "setenv PROMPT_COMMAND /bin/true" >> $SCREENRC
echo "screen -t shell bash" >> $SCREENRC
fi
# If this service doesn't already exist in the screenrc file
if ! grep $1 $SCREENRC 2>&1 > /dev/null; then
NL=`echo -ne '\015'`
echo "screen -t $1 bash" >> $SCREENRC
echo "stuff \"$2$NL\"" >> $SCREENRC
if [[ -n ${LOGDIR} ]]; then
echo "logfile ${LOGDIR}/${1}.log.${CURRENT_LOG_TIME}" >>$SCREENRC
echo "log on" >>$SCREENRC
fi
fi
}
# Stop a service in screen
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``, ``USE_SCREEN``
# screen_stop_service service
function screen_stop_service {
local service=$1
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if is_service_enabled $service; then
# Clean up the screen window
screen -S $SCREEN_NAME -p $service -X kill
fi
}
# Stop a service process
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# Uses globals ``SERVICE_DIR``, ``USE_SCREEN``
# stop_process service
function stop_process {
local service=$1
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if is_service_enabled $service; then
# Kill via pid if we have one available
if [[ -r $SERVICE_DIR/$SCREEN_NAME/$service.pid ]]; then
pkill -g $(cat $SERVICE_DIR/$SCREEN_NAME/$service.pid)
rm $SERVICE_DIR/$SCREEN_NAME/$service.pid
fi
if [[ "$USE_SCREEN" = "True" ]]; then
# Clean up the screen window
screen_stop_service $service
fi
fi
}
# Helper to get the status of each running service
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
# service_check
function service_check {
local service
local failures
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
echo "No service status directory found"
return
fi
# Check if there is any falure flag file under $SERVICE_DIR/$SCREEN_NAME
# make this -o errexit safe
failures=`ls "$SERVICE_DIR/$SCREEN_NAME"/*.failure 2>/dev/null || /bin/true`
for service in $failures; do
service=`basename $service`
service=${service%.failure}
echo "Error: Service $service is not running"
done
if [ -n "$failures" ]; then
die $LINENO "More details about the above errors can be found with screen, with ./rejoin-stack.sh"
fi
}
# Tail a log file in a screen if USE_SCREEN is true.
function tail_log {
local name=$1
local logfile=$2
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$name" "sudo tail -f $logfile"
fi
}
# Deprecated Functions
# --------------------
# _old_run_process() is designed to be backgrounded by old_run_process() to simulate a
# fork. It includes the dirty work of closing extra filehandles and preparing log
# files to produce the same logs as screen_it(). The log filename is derived
# from the service name and global-and-now-misnamed ``SCREEN_LOGDIR``
# Uses globals ``CURRENT_LOG_TIME``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
# _old_run_process service "command-line"
function _old_run_process {
local service=$1
local command="$2"
# Undo logging redirections and close the extra descriptors
exec 1>&3
exec 2>&3
exec 3>&-
exec 6>&-
if [[ -n ${SCREEN_LOGDIR} ]]; then
exec 1>&${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} 2>&1
ln -sf ${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} ${SCREEN_LOGDIR}/screen-${1}.log
# TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
export PYTHONUNBUFFERED=1
fi
exec /bin/bash -c "$command"
die "$service exec failure: $command"
}
# old_run_process() launches a child process that closes all file descriptors and
# then exec's the passed in command. This is meant to duplicate the semantics
# of screen_it() without screen. PIDs are written to
# ``$SERVICE_DIR/$SCREEN_NAME/$service.pid`` by the spawned child process.
# old_run_process service "command-line"
function old_run_process {
local service=$1
local command="$2"
# Spawn the child process
_old_run_process "$service" "$command" &
echo $!
}
# Compatibility for existing start_XXXX() functions
# Uses global ``USE_SCREEN``
# screen_it service "command-line"
function screen_it {
if is_service_enabled $1; then
# Append the service to the screen rc file
screen_rc "$1" "$2"
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$1" "$2"
else
# Spawn directly without screen
old_run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
fi
fi
}
# Compatibility for existing stop_XXXX() functions
# Stop a service in screen
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# screen_stop service
function screen_stop {
# Clean up the screen window
stop_process $1
}
# Plugin Functions
# =================
DEVSTACK_PLUGINS=${DEVSTACK_PLUGINS:-""}
# enable_plugin [branch]
#
# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
# ``url`` is a git url
# ``branch`` is a gitref. If it's not set, defaults to master
function enable_plugin {
local name=$1
local url=$2
local branch=${3:-master}
DEVSTACK_PLUGINS+=",$name"
GITREPO[$name]=$url
GITDIR[$name]=$DEST/$name
GITBRANCH[$name]=$branch
}
# fetch_plugins
#
# clones all plugins
function fetch_plugins {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Fetching DevStack plugins"
for plugin in ${plugins//,/ }; do
git_clone_by_name $plugin
done
}
# load_plugin_settings
#
# Load settings from plugins in the order that they were registered
function load_plugin_settings {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Loading plugin settings"
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
# source any known settings
if [[ -f $dir/devstack/settings ]]; then
source $dir/devstack/settings
fi
done
}
# plugin_override_defaults
#
# Run an extremely early setting phase for plugins that allows default
# overriding of services.
function plugin_override_defaults {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Overriding Configuration Defaults"
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
# source any overrides
if [[ -f $dir/devstack/override-defaults ]]; then
# be really verbose that an override is happening, as it
# may not be obvious if things fail later.
echo "$plugin has overriden the following defaults"
cat $dir/devstack/override-defaults
source $dir/devstack/override-defaults
fi
done
}
# run_plugins
#
# Run the devstack/plugin.sh in all the plugin directories. These are
# run in registration order.
function run_plugins {
local mode=$1
local phase=$2
local plugins="${DEVSTACK_PLUGINS}"
local plugin
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
if [[ -f $dir/devstack/plugin.sh ]]; then
source $dir/devstack/plugin.sh $mode $phase
fi
done
}
function run_phase {
local mode=$1
local phase=$2
if [[ -d $TOP_DIR/extras.d ]]; then
for i in $TOP_DIR/extras.d/*.sh; do
[[ -r $i ]] && source $i $mode $phase
done
fi
# the source phase corresponds to settings loading in plugins
if [[ "$mode" == "source" ]]; then
load_plugin_settings
elif [[ "$mode" == "override_defaults" ]]; then
plugin_override_defaults
else
run_plugins $mode $phase
fi
}
# Service Functions
# =================
# remove extra commas from the input string (i.e. ``ENABLED_SERVICES``)
# _cleanup_service_list service-list
function _cleanup_service_list {
echo "$1" | sed -e '
s/,,/,/g;
s/^,//;
s/,$//
'
}
# disable_all_services() removes all current services
# from ``ENABLED_SERVICES`` to reset the configuration
# before a minimal installation
# Uses global ``ENABLED_SERVICES``
# disable_all_services
function disable_all_services {
ENABLED_SERVICES=""
}
# Remove all services starting with '-'. For example, to install all default
# services except rabbit (rabbit) set in ``localrc``:
# ENABLED_SERVICES+=",-rabbit"
# Uses global ``ENABLED_SERVICES``
# disable_negated_services
function disable_negated_services {
local tmpsvcs="${ENABLED_SERVICES}"
local service
for service in ${tmpsvcs//,/ }; do
if [[ ${service} == -* ]]; then
tmpsvcs=$(echo ${tmpsvcs}|sed -r "s/(,)?(-)?${service#-}(,)?/,/g")
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
}
# disable_service() removes the services passed as argument to the
# ``ENABLED_SERVICES`` list, if they are present.
#
# For example:
# disable_service rabbit
#
# This function does not know about the special cases
# for nova, glance, and neutron built into is_service_enabled().
# Uses global ``ENABLED_SERVICES``
# disable_service service [service ...]
function disable_service {
local tmpsvcs=",${ENABLED_SERVICES},"
local service
for service in $@; do
if is_service_enabled $service; then
tmpsvcs=${tmpsvcs//,$service,/,}
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
}
# enable_service() adds the services passed as argument to the
# ``ENABLED_SERVICES`` list, if they are not already present.
#
# For example:
# enable_service qpid
#
# This function does not know about the special cases
# for nova, glance, and neutron built into is_service_enabled().
# Uses global ``ENABLED_SERVICES``
# enable_service service [service ...]
function enable_service {
local tmpsvcs="${ENABLED_SERVICES}"
local service
for service in $@; do
if ! is_service_enabled $service; then
tmpsvcs+=",$service"
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
disable_negated_services
}
# is_service_enabled() checks if the service(s) specified as arguments are
# enabled by the user in ``ENABLED_SERVICES``.
#
# Multiple services specified as arguments are ``OR``'ed together; the test
# is a short-circuit boolean, i.e it returns on the first match.
#
# There are special cases for some 'catch-all' services::
# **nova** returns true if any service enabled start with **n-**
# **cinder** returns true if any service enabled start with **c-**
# **ceilometer** returns true if any service enabled start with **ceilometer**
# **glance** returns true if any service enabled start with **g-**
# **neutron** returns true if any service enabled start with **q-**
# **swift** returns true if any service enabled start with **s-**
# **trove** returns true if any service enabled start with **tr-**
# For backward compatibility if we have **swift** in ENABLED_SERVICES all the
# **s-** services will be enabled. This will be deprecated in the future.
#
# Cells within nova is enabled if **n-cell** is in ``ENABLED_SERVICES``.
# We also need to make sure to treat **n-cell-region** and **n-cell-child**
# as enabled in this case.
#
# Uses global ``ENABLED_SERVICES``
# is_service_enabled service [service ...]
function is_service_enabled {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local enabled=1
local services=$@
local service
for service in ${services}; do
[[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && enabled=0
# Look for top-level 'enabled' function for this service
if type is_${service}_enabled >/dev/null 2>&1; then
# A function exists for this service, use it
is_${service}_enabled
enabled=$?
fi
# TODO(dtroyer): Remove these legacy special-cases after the is_XXX_enabled()
# are implemented
[[ ${service} == n-cell-* && ${ENABLED_SERVICES} =~ "n-cell" ]] && enabled=0
[[ ${service} == n-cpu-* && ${ENABLED_SERVICES} =~ "n-cpu" ]] && enabled=0
[[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && enabled=0
[[ ${service} == "cinder" && ${ENABLED_SERVICES} =~ "c-" ]] && enabled=0
[[ ${service} == "ceilometer" && ${ENABLED_SERVICES} =~ "ceilometer-" ]] && enabled=0
[[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && enabled=0
[[ ${service} == "ironic" && ${ENABLED_SERVICES} =~ "ir-" ]] && enabled=0
[[ ${service} == "neutron" && ${ENABLED_SERVICES} =~ "q-" ]] && enabled=0
[[ ${service} == "trove" && ${ENABLED_SERVICES} =~ "tr-" ]] && enabled=0
[[ ${service} == "swift" && ${ENABLED_SERVICES} =~ "s-" ]] && enabled=0
[[ ${service} == s-* && ${ENABLED_SERVICES} =~ "swift" ]] && enabled=0
done
$xtrace
return $enabled
}
# Toggle enable/disable_service for services that must run exclusive of each other
# $1 The name of a variable containing a space-separated list of services
# $2 The name of a variable in which to store the enabled service's name
# $3 The name of the service to enable
function use_exclusive_service {
local options=${!1}
local selection=$3
local out=$2
[ -z $selection ] || [[ ! "$options" =~ "$selection" ]] && return 1
local opt
for opt in $options;do
[[ "$opt" = "$selection" ]] && enable_service $opt || disable_service $opt
done
eval "$out=$selection"
return 0
}
# System Functions
# ================
# Only run the command if the target file (the last arg) is not on an
# NFS filesystem.
function _safe_permission_operation {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local args=( $@ )
local last
local sudo_cmd
local dir_to_check
let last="${#args[*]} - 1"
local dir_to_check=${args[$last]}
if [ ! -d "$dir_to_check" ]; then
dir_to_check=`dirname "$dir_to_check"`
fi
if is_nfs_directory "$dir_to_check" ; then
$xtrace
return 0
fi
if [[ $TRACK_DEPENDS = True ]]; then
sudo_cmd="env"
else
sudo_cmd="sudo"
fi
$xtrace
$sudo_cmd $@
}
# Exit 0 if address is in network or 1 if address is not in network
# ip-range is in CIDR notation: 1.2.3.4/20
# address_in_net ip-address ip-range
function address_in_net {
local ip=$1
local range=$2
local masklen=${range#*/}
local network=$(maskip ${range%/*} $(cidr2netmask $masklen))
local subnet=$(maskip $ip $(cidr2netmask $masklen))
[[ $network == $subnet ]]
}
# Add a user to a group.
# add_user_to_group user group
function add_user_to_group {
local user=$1
local group=$2
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
# SLE11 and openSUSE 12.2 don't have the usual usermod
if ! is_suse || [[ "$os_VENDOR" = "openSUSE" && "$os_RELEASE" != "12.2" ]]; then
sudo usermod -a -G "$group" "$user"
else
sudo usermod -A "$group" "$user"
fi
}
# Convert CIDR notation to a IPv4 netmask
# cidr2netmask cidr-bits
function cidr2netmask {
local maskpat="255 255 255 255"
local maskdgt="254 252 248 240 224 192 128"
set -- ${maskpat:0:$(( ($1 / 8) * 4 ))}${maskdgt:$(( (7 - ($1 % 8)) * 4 )):3}
echo ${1-0}.${2-0}.${3-0}.${4-0}
}
# Gracefully cp only if source file/dir exists
# cp_it source destination
function cp_it {
if [ -e $1 ] || [ -d $1 ]; then
cp -pRL $1 $2
fi
}
# HTTP and HTTPS proxy servers are supported via the usual environment variables [1]
# ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in
# ``localrc`` or on the command line if necessary::
#
# [1] http://www.w3.org/Daemon/User/Proxies/ProxyClients.html
#
# http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
# Unused function
#function export_proxy_variables {
# if isset http_proxy ; then
# export http_proxy=$http_proxy
# fi
# if isset https_proxy ; then
# export https_proxy=$https_proxy
# fi
# if isset no_proxy ; then
# export no_proxy=$no_proxy
# fi
#}
# Returns true if the directory is on a filesystem mounted via NFS.
function is_nfs_directory {
local mount_type=`stat -f -L -c %T $1`
test "$mount_type" == "nfs"
}
# Return the network portion of the given IP address using netmask
# netmask is in the traditional dotted-quad format
# maskip ip-address netmask
function maskip {
local ip=$1
local mask=$2
local l="${ip%.*}"; local r="${ip#*.}"; local n="${mask%.*}"; local m="${mask#*.}"
local subnet=$((${ip%%.*}&${mask%%.*})).$((${r%%.*}&${m%%.*})).$((${l##*.}&${n##*.})).$((${ip##*.}&${mask##*.}))
echo $subnet
}
# Service wrapper to restart services
# restart_service service-name
function restart_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 restart
else
sudo /sbin/service $1 restart
fi
}
# Only change permissions of a file or directory if it is not on an
# NFS filesystem.
function safe_chmod {
_safe_permission_operation chmod $@
}
# Only change ownership of a file or directory if it is not on an NFS
# filesystem.
function safe_chown {
_safe_permission_operation chown $@
}
# Service wrapper to start services
# start_service service-name
function start_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 start
else
sudo /sbin/service $1 start
fi
}
# Service wrapper to stop services
# stop_service service-name
function stop_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 stop
else
sudo /sbin/service $1 stop
fi
}
# Restore xtrace
$XTRACE
# Local variables:
# mode: shell-script
# End:
if [[ $EUID -gt 0 ]]; then
echo "####################################################"
echo "# #"
echo "# ERROR: You must be root to run this script!!!! #"
echo "# #"
echo "####################################################"
exit 1
fi
export PATH=$PATH:/usr/local/bin:/usr/local/sbin
# XXX a bit brutal but otherwise it cannot work, without tty enabled.
if [ -f /etc/sudoers ]; then
sed -i "/requiretty/d" /etc/sudoers
fi
# Include Additional Functions
function download_playbook {
if [ ! -f /etc/opt/slapcache.cfg ]; then
slapcache-conf
fi
DFILE="/tmp/tmpplaybook$(basename $0).$$/"
TFILE="archive.tar.gz"
mkdir -p $DFILE
cd $DFILE
slapcache-download --destination=$TFILE
tar -xzvf $TFILE
rm $TFILE
}
# Determine what system we are running on. This provides ``os_VENDOR``,
# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
# and ``DISTRO``
GetDistro
if [[ ${DISTRO} =~ (-.) ]] && [[ -f /etc/debian_version ]]; then
apt_get install lsb-release
GetDistro
fi
# Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``export FORCE=yes``
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then
echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes"
fi
fi
# Make sure wheezy backports are available.
if [[ $DISTRO == "wheezy" ]]; then
echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
fi
if is_fedora && [[ $DISTRO == "rhel7" ]]; then
# RHEL requires EPEL for many Ansible dependencies
# NOTE: We always remove and install latest -- some environments
# use snapshot images, and if EPEL version updates they break
# unless we update them to latest version.
if sudo yum repolist enabled epel | grep -q 'epel'; then
uninstall_package epel-release || true
fi
# This trick installs the latest epel-release from a bootstrap
# repo, then removes itself (as epel-release installed the
# "real" repo).
#
# You would think that rather than this, you could use
# $releasever directly in .repo file we create below. However
# RHEL gives a $releasever of "6Server" which breaks the path;
# see https://bugzilla.redhat.com/show_bug.cgi?id=1150759
cat < $ANSIBLE_PLUGIN_LOCATION/log_parse.py
import os
import time
import json
import ansible
baseModule = object
ANSIBLE_VERSION = 1
if hasattr(ansible, 'plugins') and hasattr(ansible.plugins, 'callback'):
baseModule = ansible.plugins.callback.CallbackBase
ANSIBLE_VERSION = 2
class CallbackModule(baseModule):
"""
logs playbook results, per host, in /var/log/ansible/hosts
"""
log_path = '/var/log/ansible/hosts'
fd_list = {}
def __init__(self):
if ANSIBLE_VERSION > 1:
super(CallbackModule, self).__init__()
if not os.path.exists(self.log_path):
os.makedirs(self.log_path)
else:
for filename in os.listdir(self.log_path):
filepath = os.path.join(self.log_path, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.unlink(filepath)
def writeLog(self, host, category, content):
if not self.fd_list.has_key(category):
self.fd_list[category] = open(
os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
)
self.fd_list[category].write(content + '\n')
def log(self, host, category, data, ignore_errors=False):
if host == "localhost":
host = "127.0.0.1" # keep compatibility
if type(data) == dict:
if '_ansible_verbose_override' in data:
# avoid logging extraneous data
return
data = data.copy()
content = json.dumps(data)
if ignore_errors:
category = '%s_IGNORED' % category
self.writeLog(host, category, content)
def _stats(self, stats):
for key in self.fd_list:
self.fd_list[key].close()
def runner_on_failed(self, host, res, ignore_errors=False):
self.log(host, 'FAILED', res, ignore_errors)
def runner_on_ok(self, host, res):
self.log(host, 'OK', res)
def runner_on_skipped(self, host, item=None):
pass
def runner_on_unreachable(self, host, res):
self.log(host, 'UNREACHABLE', res)
def runner_on_async_failed(self, host, res, jid):
self.log(host, 'ASYNC_FAILED', res)
def playbook_on_import_for_host(self, host, imported_file):
self.log(host, 'IMPORTED', imported_file)
def playbook_on_not_import_for_host(self, host, missing_file):
self.log(host, 'NOTIMPORTED', missing_file)
def playbook_on_stats(self, stats):
self._stats(stats)
EOF
ansible localhost -m easy_install -a name=slapcache --connection=local
ansible localhost -m easy_install -a name=requests --connection=local
# Include Additional Functions
if [ ! -f /etc/opt/slapcache.cfg ]; then
slapcache-conf
fi
sed -i "s/key = slapos-global-key/key = slapos-global-official-vifib-key/g" /etc/opt/slapcache.cfg
DFILE="/tmp/tmpplaybook_unstable$(basename $0).$$/"
TFILE="archive.tar.gz"
mkdir -p $DFILE
cd $DFILE
wget -O $TFILE http://10.0.2.100/playbook.tar.gz
tar -xzvf $TFILE
rm $TFILE
clear
echo "Starting Ansible playbook:"
ansible-playbook $TEST_YML_PATH -i hosts --connection=local
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/dummy/ 0000775 0000000 0000000 00000000000 13712426147 0031020 5 ustar 00root root 0000000 0000000 README.md 0000664 0000000 0000000 00000000570 13712426147 0032222 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/dummy # Dummy Software Release
The main purpose of this Software Release is to be used in tests !
As a consequence, any change to it must be tied with corresponding changes in
tests using this Software Release.
Dummy has be written with the purpose to be fast, so it embeeds the
minimum set of needed features. Please, do not change anything of it
without any strong motivation.
instance.cfg 0000664 0000000 0000000 00000001046 13712426147 0033227 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/dummy [buildout]
parts =
log-writer
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[directory]
recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
bin = $${:home}/bin
etc = $${:home}/etc
service = $${:etc}/service
script = $${:etc}/run
var = $${:home}/var
log = $${:var}/log
[log-writer]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
echo "Hello : $(date)" >> $${directory:log}/log.log
rendered = $${directory:script}/log-writer
mode = 700
software.cfg 0000664 0000000 0000000 00000000523 13712426147 0033254 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/dummy [buildout]
extends =
../../../../stack/slapos.cfg
parts =
instance-template
slapos-cookbook
[instance-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644
[versions]
slapos.recipe.template = 4.4
instance-kvm-resilient-test.cfg.jinja2 0000664 0000000 0000000 00000005657 13712426147 0037030 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite [buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
request-resilient-instance
deploy-unit-test
deploy-scalability-test
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc/
var = ${:home}/var/
srv = ${:home}/srv/
bin = ${:home}/bin/
tmp = ${:home}/tmp/
log = ${:var}/log/
services = ${:etc}/service/
scripts = ${:etc}/run/
[deploy-scalability-test]
recipe = slapos.cookbook:wrapper
# XXX: put it in etc/run in case of scalability test so that it runs automatically.
wrapper-path = ${directory:bin}/runKVMResiliencyTestSuite
testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm root_instance_name='${request-resilient-instance:name}'
command-line = {{ bin_directory }}/runResiliencyScalabilityTestNode ${:testnode-parameters} ${:test-parameters}
[deploy-unit-test]
recipe = collective.recipe.template
test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm root_instance_name='${request-resilient-instance:name}'
input = inline:
#!/bin/sh
exec {{ bin_directory }}/runResiliencyUnitTestTestNode $@ ${:test-parameters}
output = ${directory:bin}/runTestSuite
mode = 755
[request-resilient-instance]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm-resilient
name = Resilient Instance (Root Instance)
{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
config-virtual-hard-drive-gzipped = ${slap-parameter:virtual-hard-drive-gzipped}
config-resiliency-backup-periodicity = */5 * * * *
config-resilient-clone-number = 1
config-ignore-known-hosts-file = false
return = ipv6
# XXX What to do?
sla-computer_guid = ${slap-connection:computer-id}
[slap-parameter]
virtual-hard-drive-url = {{ default_test_image_url }}
virtual-hard-drive-md5sum = {{ default_test_image_md5sum }}
virtual-hard-drive-gzipped = true
instance-resilient-test.cfg.jinja2 0000664 0000000 0000000 00000005661 13712426147 0036230 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite [buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
request-resilient-instance
deploy-unit-test
deploy-scalability-test
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc/
var = ${:home}/var/
srv = ${:home}/srv/
bin = ${:home}/bin/
tmp = ${:home}/tmp/
log = ${:var}/log/
services = ${:etc}/service/
scripts = ${:etc}/run/
[deploy-scalability-test]
recipe = slapos.cookbook:wrapper
testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=runner root_instance_name='${request-resilient-instance:name}'
command-line = {{ bin_directory }}/runResiliencyScalabilityTestNode ${:testnode-parameters} ${:test-parameters}
wrapper-path = ${directory:scripts}/runResiliencyTestSuite
[deploy-unit-test]
recipe = collective.recipe.template
test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=runner root_instance_name='${request-resilient-instance:name}'
input = inline:
#!/bin/sh
exec {{ bin_directory }}/runResiliencyUnitTestTestNode $@ ${:test-parameters}
output = ${directory:bin}/runTestSuite
mode = 755
[request-resilient-instance]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = resilient
name = Resilient Instance (Root Instance)
{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-resiliency-backup-periodicity = */10 * * * *
config-resilient-clone-number = 1
config-ignore-known-hosts-file = false
config-cpu-usage-ratio = 1
# Use one external folder (same of erp5testnode for build the software
# This is a way to preserve the latest software release build speeding up
# the tests.)
config-software-root = ${buildout:directory}/../../soft
config-buildout-shared-folder = ${buildout:directory}/../../shared
config-no-ipv4-frontend = true
# XXX hardcoded
#config-frontend-domain = google.com
# XXX Hack to deploy Root Instance on the same computer as the type-test Instance
sla-computer_guid = ${slap-connection:computer-id}
return = backend-url
[slap-parameter]
instance.cfg.in 0000664 0000000 0000000 00000001423 13712426147 0032500 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite [buildout]
extends = ${template:output}
[switch_softwaretype]
default = $${:test}
RootSoftwareInstance = $${:test}
# Used for the test of resiliency. The system wants a "test" software_type.
test = dynamic-template-resilient-test:rendered
[dynamic-template-resilient-test]
recipe = slapos.recipe.template:jinja2
template = ${template-resilient-test:location}/${template-resilient-test:filename}
rendered = $${buildout:directory}/template-resilient-test.cfg
bin-directory = ${buildout:bin-directory}
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
${template-resilient-test:extra-context}
mode = 0644
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite/kvm.cfg 0000664 0000000 0000000 00000001513 13712426147 0031143 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../kvm/development.cfg
testsuite.cfg
parts += template-erp5testnode
[default-test-image]
recipe = hexagonit.recipe.download
ignore-existing = true
filename = ${:_buildout_section_name_}
url = http://www.nexedi.org/static/slapos/kvm_resiliency_test/virtual.qcow.gz
md5sum = dd82c771f6f7738fb4b0fc1330ed8236
download-only = true
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-resilient-test]
filename = instance-kvm-resilient-test.cfg.jinja2
md5sum = 71ddbdeb8769bcb0ebb3c9407ef7e36c
# Ingest extra-context, on a the final template-resilient-test rendering
# always ingest raw values.
extra-context =
raw default_test_image_url file://${default-test-image:location}/${default-test-image:filename}
raw default_test_image_md5sum ${default-test-image:md5sum}
slaprunner.cfg 0000664 0000000 0000000 00000000505 13712426147 0032460 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite [buildout]
extends =
../../slaprunner/development.cfg
testsuite.cfg
parts += template-erp5testnode
[template-resilient-test]
filename = instance-resilient-test.cfg.jinja2
md5sum = cf34be404bdd93a72df9c593f8762cb2
[exporter-default-configuration]
# Define shorter interaction to speed up tests
backup_wait_time = 1
testsuite.cfg 0000664 0000000 0000000 00000000723 13712426147 0032322 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/erp5testnode/testsuite [template-erp5testnode]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = d864a32edab3c4f7207a7d4fe6bb7e30
output = ${buildout:directory}/template.cfg
mode = 0644
[template]
output = ${buildout:directory}/template-original.cfg
[template-resilient-test]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
mode = 0644
download-only = true
on-update = true
extra-context =
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/ 0000775 0000000 0000000 00000000000 13712426147 0025742 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/buildout.hash.cfg 0000664 0000000 0000000 00000001047 13712426147 0031176 0 ustar 00root root 0000000 0000000 [template]
_update_hash_filename_ = instance.cfg
md5sum = 7ab2a242df988bf5c10bf8002acac3bd
[instance-etherpad-lite]
_update_hash_filename_ = instance-etherpad-lite.cfg
md5sum = 7ead62829a85f5922bb9ed0fc8dfb0df
[template-conf]
_update_hash_filename_ = templates/settings.json.in
md5sum = 19ab39e6b3256c82fd54ce074488b136
[template-run-script]
_update_hash_filename_ = templates/run.sh.in
md5sum = eac870b5f30e735e109a48913af2fae3
[template-deps-script]
_update_hash_filename_ = templates/installDeps.sh.in
md5sum = 53d0d53d419bd9ee592d3e1a1c84c758
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/common.cfg 0000664 0000000 0000000 00000004327 13712426147 0027721 0 ustar 00root root 0000000 0000000 [buildout]
versions = versions
extends =
../../stack/slapos.cfg
../../component/gcc/buildout.cfg
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/gzip/buildout.cfg
../../component/git/buildout.cfg
../../component/nodejs/buildout.cfg
../../component/postgresql/buildout.cfg
./buildout.hash.cfg
parts =
postgresql
nodejs
etherpad-lite-repository
install-deps
template
lxml-python
eggs
instance-recipe-egg
unzip= true
[eggs]
recipe = zc.recipe.egg
eggs =
[instance-recipe]
egg = slapos.cookbook
module = etherpad-lite
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[etherpad-lite-repository]
recipe = slapos.recipe.build:gitclone
repository = http://github.com/ether/etherpad-lite.git
branch = develop
git-executable = ${git:location}/bin/git
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template.cfg
mode = 0644
[instance-etherpad-lite]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template-etherpad-lite.cfg
mode = 0644
[template-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-run-script]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-deps-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:_update_hash_filename_}
etherpad-location = ${etherpad-lite-repository:location}
nodejs-location = ${nodejs:location}
curl-location = ${curl:location}
postgre-location = ${postgresql:location}
mode = 0755
output = ${etherpad-lite-repository:location}/bin/installDeps.sh
[install-deps]
recipe = plone.recipe.command
command = ${template-deps-script:output}
update-command = command
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/development.cfg 0000664 0000000 0000000 00000001522 13712426147 0030745 0 ustar 00root root 0000000 0000000 [buildout]
extends =
common.cfg
parts +=
slapos.cookbook-repository
slapos.toolbox-repository
check-recipe
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
branch = etherpad-lite
git-executable = ${git:location}/bin/git
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = master
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
instance-etherpad-lite.cfg 0000664 0000000 0000000 00000005116 13712426147 0032700 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite [buildout]
parts =
etherpad-lite
publish-connection-informations
# frontend-etherpad
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:etc}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
etherpad-conf = $${rootdirectory:etc}/etherpad/
etherpad-repository-location = $${buildout:directory}/parts/etherpad-lite-repository
[publish-connection-informations]
recipe = slapos.cookbook:publish
url = $${request-frontend:connection-site_url}
[etherpad-conf-generation]
recipe = slapos.recipe.template
url = ${template-conf:location}/${template-conf:filename}
ip = $${slap-network-information:global-ipv6}
dirtydb-location = $${rootdirectory:var}/dirty.db
port = 9001
mode = 0644
output = $${directory:etherpad-conf}/settings.json
[etherpad-run-script]
recipe = slapos.recipe.template
url = ${template-run-script:location}/${template-run-script:filename}
etherpad-location = ${etherpad-lite-repository:location}
etherpad-repository-location = $${directory:etherpad-repository-location}
nodejs-location = ${nodejs:location}
etherpad-deps-script-location = ${template-deps-script:output}
etherpad-conf-location = $${etherpad-conf-generation:output}
etherpad-conf-name = settings-$${slap-connection:partition-id}.json
mode = 0755
output = $${rootdirectory:bin}/run.sh
# Command line comes from the run script of etherpad-lite
[etherpad-lite]
recipe = slapos.cookbook:wrapper
wrapper-path = $${basedirectory:run}/etherpad-lite
command-line = $${etherpad-run-script:output} -s $${etherpad-run-script:etherpad-conf-name}
[request-frontend]
<= slap-connection
recipe = slapos.cookbook:request
name = Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
return = site_url
[frontend-etherpad]
recipe = slapos.cookbook:check_url_available
path = $${basedirectory:promises}/frontend-etherpad
url = $${request-frontend:connection-site_url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/instance.cfg 0000664 0000000 0000000 00000000412 13712426147 0030224 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch_softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-etherpad-lite:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/software.cfg 0000664 0000000 0000000 00000001522 13712426147 0030255 0 ustar 00root root 0000000 0000000 [buildout]
extends =
common.cfg
parts +=
slapos.cookbook-repository
slapos.toolbox-repository
check-recipe
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
branch = etherpad-lite
git-executable = ${git:location}/bin/git
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = master
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/templates/ 0000775 0000000 0000000 00000000000 13712426147 0027740 5 ustar 00root root 0000000 0000000 installDeps.sh.in 0000775 0000000 0000000 00000003354 13712426147 0033114 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/templates #!/bin/sh
# Ugly setup
cd ${:etherpad-location}
export PATH=$PATH:${:postgre-location}/bin
#Is gnu-grep (ggrep) installed on SunOS (Solaris)
if [ $(uname) = "SunOS" ]; then
hash ggrep > /dev/null 2>&1 || {
echo "Please install ggrep (pkg install gnu-grep)" >&2
exit 1
}
fi
#Is wget installed?
hash ${:curl-location}/bin/curl > /dev/null 2>&1 || {
echo "Please install curl" >&2
exit 1
}
#Is node installed?
hash ${:nodejs-location}/bin/node > /dev/null 2>&1 || {
echo "Please install node.js ( http://nodejs.org )" >&2
exit 1
}
#Is npm installed?
hash ${:nodejs-location}/bin/npm > /dev/null 2>&1 || {
echo "Please install npm ( http://npmjs.org )" >&2
exit 1
}
echo "Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient."
(
mkdir -p node_modules
cd node_modules
[ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
cd ep_etherpad-lite
${:nodejs-location}/bin/npm install --loglevel warn
) || {
rm -rf node_modules
exit 1
}
echo "Ensure jQuery is downloaded and up to date..."
DOWNLOAD_JQUERY="true"
NEEDED_VERSION="1.9.1"
if [ $DOWNLOAD_JQUERY = "true" ]; then
${:curl-location}/bin/curl -lo src/static/js/jquery.js http://code.jquery.com/jquery-$NEEDED_VERSION.js || exit 1
fi
#Remove all minified data to force node creating it new
echo "Clear minfified cache..."
rm -f var/minified*
echo "ensure custom css/js files are created..."
for f in "index" "pad" "timeslider"
do
if [ ! -f "src/static/custom/$f.js" ]; then
cp "src/static/custom/js.template" "src/static/custom/$f.js" || exit 1
fi
if [ ! -f "src/static/custom/$f.css" ]; then
cp "src/static/custom/css.template" "src/static/custom/$f.css" || exit 1
fi
done
exit 0
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/templates/run.sh.in 0000775 0000000 0000000 00000001020 13712426147 0031501 0 ustar 00root root 0000000 0000000 #!/bin/sh
# Copy repository content from the software release
mkdir ${:etherpad-repository-location}
cp -R ${:etherpad-location}/* ${:etherpad-repository-location}
#Move to the folder where ep-lite is installed
cd ${:etherpad-repository-location}
# XXX Vivien: very ugly, definitely need to find a cleaner way
cp ${:etherpad-conf-location} ${:etherpad-repository-location}/${:etherpad-conf-name}
#Move to the node folder and start
echo "start..."
${:nodejs-location}/bin/node node_modules/ep_etherpad-lite/node/server.js $*
settings.json.in 0000664 0000000 0000000 00000011577 13712426147 0033034 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/templates /*
This file must be valid JSON. But comments are allowed
Please edit settings.json, not settings.json.template
*/
{
// Name your instance!
"title": "Etherpad",
// favicon default name
// alternatively, set up a fully specified Url to your own favicon
"favicon": "favicon.ico",
//IP and port which etherpad should bind at
"ip": "${:ip}",
"port" : "${:port}",
// Session Key, used for reconnecting user sessions
// Set this to a secure string at least 10 characters long. Do not share this value.
"sessionKey" : "",
/*
// Node native SSL support
// this is disabled by default
//
// make sure to have the minimum and correct file access permissions set
// so that the Etherpad server can access them
"ssl" : {
"key" : "/path-to-your/epl-server.key",
"cert" : "/path-to-your/epl-server.crt"
},
*/
//The Type of the database. You can choose between dirty, postgres, sqlite and mysql
//You shouldn't use "dirty" for for anything else than testing or development
"dbType" : "dirty",
//the database specific settings
"dbSettings" : {
"filename" : "${:dirtydb-location}"
},
/* An Example of MySQL Configuration
"dbType" : "mysql",
"dbSettings" : {
"user" : "root",
"host" : "localhost",
"password": "",
"database": "store"
},
*/
//the default text of a pad
"defaultPadText" : "Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at http:\/\/etherpad.org\n",
/* Users must have a session to access pads. This effectively allows only group pads to be accessed. */
"requireSession" : false,
/* Users may edit pads but not create new ones. Pad creation is only via the API. This applies both to group pads and regular pads. */
"editOnly" : false,
/* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly,
but makes it impossible to debug the javascript/css */
"minify" : true,
/* How long may clients use served javascript code (in seconds)? Without versioning this
may cause problems during deployment. Set to 0 to disable caching */
"maxAge" : 21600, // 60 * 60 * 6 = 6 hours
/* This is the path to the Abiword executable. Setting it to null, disables abiword.
Abiword is needed to advanced import/export features of pads*/
"abiword" : null,
/* This setting is used if you require authentication of all users.
Note: /admin always requires authentication. */
"requireAuthentication": false,
/* Require authorization by a module, or a user with is_admin set, see below. */
"requireAuthorization": false,
/* Users for basic authentication. is_admin = true gives access to /admin.
If you do not uncomment this, /admin will not be available! */
/*
"users": {
"admin": {
"password": "changeme1",
"is_admin": true
},
"user": {
"password": "changeme1",
"is_admin": false
}
},
*/
// restrict socket.io transport methods
"socketTransportProtocols" : ["xhr-polling"],
/* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */
"loglevel": "INFO",
//Logging configuration. See log4js documentation for further information
// https://github.com/nomiddlename/log4js-node
// You can add as many appenders as you want here:
"logconfig" :
{ "appenders": [
{ "type": "console",
// "category": "access"// only logs pad access
}
/*
, { "type": "file"
, "filename": "your-log-file-here.log"
, "maxLogSize": 1024
, "backups": 3 // how many log files there're gonna be at max
//, "category": "test" // only log a specific category
}*/
/*{ "type": "logLevelFilter",
"level": "warn" // filters out all log messages that have a lower level than "error"
, "appender":
{ Use whatever appender you want here }
}*/
/*
, { "type": "logLevelFilter"
, "level": "error" // filters out all log messages that have a lower level than "error"
, "appender":
{ "type": "smtp"
, "subject": "An error occured in your EPL instance!"
, "recipients": "bar@blurdybloop.com, baz@blurdybloop.com"
, "sendInterval": 60*5 // in secs -- will buffer log messages; set to 0 to send a mail for every message
, "transport": "SMTP", "SMTP": { // see https://github.com/andris9/Nodemailer#possible-transport-methods
"host": "smtp.example.com", "port": 465,
"secureConnection": true,
"auth": {
"user": "foo@example.com",
"pass": "bar_foo"
}
}
}
}*/
] }
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/etherpad-lite/todo.rst 0000664 0000000 0000000 00000000236 13712426147 0027442 0 ustar 00root root 0000000 0000000 TODO:
+ Modify etherpad-lite to read absolute path correctly, so that we aren't
forced to move everything to the software release directory for it to work
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/fluentd/ 0000775 0000000 0000000 00000000000 13712426147 0024654 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/fluentd/instance-input-schema.json0000664 0000000 0000000 00000000545 13712426147 0031752 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate Fluentd",
"additionalProperties": false,
"properties": {
"conf_text": {
"description": "Fluentd configuration. You can write it entirely by yourself. See fluentd-agent.conf.jinja2.in",
"default": "",
"type": "string"
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/fluentd/instance.cfg.in 0000664 0000000 0000000 00000002305 13712426147 0027546 0 ustar 00root root 0000000 0000000 [buildout]
parts =
service-fluentd
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap_connection:computer_id}
partition = ${slap_connection:partition_id}
url = ${slap_connection:server_url}
key = ${slap_connection:key_file}
cert = ${slap_connection:cert_file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
script = ${:etc}/run/
service = ${:etc}/service
promise = ${:etc}/promise/
log = ${:var}/log
bin = ${:home}/bin
[service-fluentd]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/fluentd-service
command-line = {{ fluentd_location }}/bin/fluentd
-v
-c ${fluentd-agent-conf:rendered}
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-agent-conf]
recipe = slapos.recipe.template:jinja2
template = inline:{% raw -%}
{{ slapparameter_dict.get('conf_text', '') }}
{%- endraw %}
rendered = ${directory:etc}/fluentd-agent.conf
mode = 0644
context =
key slapparameter_dict instance-parameter:configuration
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/fluentd/software.cfg 0000664 0000000 0000000 00000001050 13712426147 0027163 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg
parts =
instance-profile
slapos-cookbook
[instance-profile]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/instance.cfg
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
key fluentd_location fluentd:location
[fluentd]
gems +=
fluent-plugin-wendelin==0.4
fluent-plugin-bin
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/ 0000775 0000000 0000000 00000000000 13712426147 0024635 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/buildout.hash.cfg 0000664 0000000 0000000 00000000656 13712426147 0030076 0 ustar 00root root 0000000 0000000 [template]
_update_hash_filename_ = instance.cfg.in
md5sum = 428669a609aca3e0a7cae1387d332a75
[template-gateone]
filename = instance-gateone.cfg.in
md5sum = 7c4cb83d6ccf5a5bcdb7b972123ac3e4
[template-logrotate-base]
_update_hash_filename_ = instance-logrotate-base.cfg.in
md5sum = ebee34c726f30fff95a9e0ea167a635f
[template-nginx-conf]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = fe055886e255d9dea9f9f29008f6e780
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/instance-gateone.cfg.in 0000664 0000000 0000000 00000011672 13712426147 0031156 0 ustar 00root root 0000000 0000000
{% set python_bin = parameter_dict['python-executable'] -%}
{% set publish_dict = {} -%}
{% set part_list = [] -%}
{% set ipv6 = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
tmp = ${buildout:directory}/tmp
log = ${:var}/log
services = ${:etc}/service
script = ${:etc}/run
promises = ${:etc}/promise
run = ${:var}/run
ca-dir = ${:etc}/ssl
requests = ${:ca-dir}/requests/
private = ${:ca-dir}/private/
certs = ${:ca-dir}/certs/
newcerts = ${:ca-dir}/newcerts/
crl = ${:ca-dir}/crl/
[gateone-dir]
gateone = ${directory:srv}/gateone
log-prefix = ${directory:log}/gateone
cache = ${:gateone}/cache
sessions = ${:gateone}/sessions
users = ${:gateone}/users
conf = ${directory:etc}/gateone
ssl = ${:conf}/ssl
[gateone-configure]
recipe = slapos.cookbook:wrapper
port = 10443
ipv6 = {{ ipv6 }}
ipv4 = {{ ipv4 }}
settings-dir = ${gateone-dir:conf}
wrapper-path = ${directory:bin}/gateone-configure
# XXX- issue with dtach=true, we set --dtach=false
command =
{{ parameter_dict['gateone-bin'] }} --address=${:ipv4} --port=${:port} --certificate=${gateone-dir:ssl}/certificate.pem --keyfile=${gateone-dir:ssl}/keyfile.pem --cache_dir=${gateone-dir:cache} --pid_file=${directory:run}/gateone.pid --session_dir=${gateone-dir:sessions} --settings_dir=${:settings-dir} --user_dir=${gateone-dir:users} --log_file_prefix=${gateone-dir:log-prefix} --origins=${:ipv4} --logging=info --dtach=false
#--gid=1012 --uid=987
command-line =
${:command} --configure
environment =
PATH={{ bin_directory }}:{{ openssl_location }}/bin:{{ parameter_dict['dtach-location'] }}/bin:/usr/local/bin:/usr/bin:/bin
LD_LIBRARY_PATH={{ parameter_dict['readline-location'] }}/lib
[gateone-run]
recipe = slapos.cookbook:wrapper
port = 10443
ipv6 = {{ ipv6 }}
ipv4 = {{ ipv4 }}
wrapper-path = ${directory:services}/gateone
command-line = ${gateone-configure:command}
environment =
PATH={{ bin_directory }}:{{ openssl_location }}/bin:{{ parameter_dict['dtach-location'] }}/bin:/usr/local/bin:/usr/bin:/bin
LD_LIBRARY_PATH={{ parameter_dict['readline-location'] }}/lib
HOME=${buildout:directory}
PWD=${gateone-dir:gateone}
# --cookie_secret 45-characters
# --uid needs to be set for slapos
# --origins=${:ipv4};${:ipv6}
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_location }}/bin/openssl
ca-dir = ${directory:ca-dir}
requests-directory = ${directory:requests}
wrapper = ${directory:services}/certificate_authority
ca-private = ${directory:private}
ca-certs = ${directory:certs}
ca-newcerts = ${directory:newcerts}
ca-crl = ${directory:crl}
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = ${directory:certs}/nginx.key
cert-file = ${directory:certs}/nginx.crt
executable = ${nginx-run:wrapper-path}
wrapper = ${directory:services}/nginx-proxy
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
client_body_temp_path = ${directory:tmp}/client_body_temp_path
proxy_temp_path = ${directory:tmp}/proxy_temp_path
fastcgi_temp_path = ${directory:tmp}/fastcgi_temp_path
uwsgi_temp_path = ${directory:tmp}/uwsgi_temp_path
scgi_temp_path = ${directory:tmp}/scgi_temp_path
[nginx-config-dict]
nb_workers = 2
ipv6 = {{ ipv6 }}
ipv4 = {{ ipv4 }}
port = 10443
backend-url = https://${gateone-configure:ipv4}:${gateone-configure:port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
pid = ${directory:run}/nginx.pid
log = ${directory:log}/nginx.log
access-log = ${directory:log}/nginx.access.log
error-log = ${directory:log}/nginx.error.log
tmp = $${directory:tmp}/
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-ngnix-conf'] }}
rendered = ${directory:etc}/nginx.conf
context =
section parameter_dict nginx-config-dict
section param_tempdir tempdirectory
[nginx-run]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/nginx_proxy
command-line = {{ parameter_dict['nginx-location'] }}/sbin/nginx -p ${buildout:directory} -c ${nginx-conf:rendered}
[logrotate-apache]
< = logrotate-entry-base
name = apache
log = ${apache-conf:error-log} ${apache-conf:access-log}
post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
[logrotate-entry-base]
recipe = slapos.cookbook:logrotate.d
logrotate-entries = ${logrotate:logrotate-entries}
backup = ${logrotate:backup}
[publish]
recipe = slapos.cookbook:publish
url = https://[${nginx-config-dict:ipv6}]:${gateone-run:port}
[buildout]
extends =
{{ logrotate_cfg }}
parts =
certificate-authority
ca-nginx
publish
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameter]
instance-logrotate-base.cfg.in 0000664 0000000 0000000 00000003016 13712426147 0032356 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone [buildout]
parts =
cron-entry-logrotate
[cron]
recipe = slapos.cookbook:cron
cron-entries = ${logrotate-directory:cron-entries}
dcrond-binary = {{ dcron_location }}/sbin/crond
crontabs = ${logrotate-directory:crontabs}
cronstamps = ${logrotate-directory:cronstamps}
catcher = ${cron-simplelogger:wrapper}
binary = ${logrotate-directory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = ${logrotate-directory:bin}/cron_simplelogger
log = ${logrotate-directory:log}/cron.log
[logrotate]
recipe = slapos.cookbook:logrotate
logrotate-entries = ${logrotate-directory:logrotate-entries}
backup = ${logrotate-directory:logrotate-backup}
logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
gzip-binary = {{ gzip_location }}/bin/gzip
gunzip-binary = {{ gzip_location }}/bin/gunzip
wrapper = ${logrotate-directory:bin}/logrotate
conf = ${logrotate-directory:etc}/logrotate.conf
state-file = ${logrotate-directory:srv}/logrotate.status
[cron-entry-logrotate]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = logrotate
frequency = 0 0 * * *
command = ${logrotate:wrapper}
[logrotate-directory]
recipe = slapos.cookbook:mkdirectory
cron-entries = ${:etc}/cron.d
cronstamps = ${:etc}/cronstamps
crontabs = ${:etc}/crontabs
logrotate-backup = ${:backup}/logrotate
logrotate-entries = ${:etc}/logrotate.d
bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
backup = ${:srv}/backup
etc = ${buildout:directory}/etc
services = ${:etc}/service
log = ${buildout:directory}/var/log
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/instance.cfg.in 0000664 0000000 0000000 00000003357 13712426147 0027537 0 ustar 00root root 0000000 0000000 [buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extra-context =
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key ipv6_set slap-configuration:ipv6
key ipv4_set slap-configuration:ipv4
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
raw logrotate_cfg {{ template_logrotate_base }}
raw dash_binary {{ dash_location }}/bin/dash
raw bin_directory {{ bin_directory }}
raw openssl_location {{ openssl_location }}
${:extra-context}
[dynamic-template-gateone-parameters]
bin-directory = {{ bin_directory }}
#python-executable =
readline-location = {{ readline_location }}
dtach-location = {{ dtach_location }}
gateone-location = {{ gateone_location}}
gateone-bin = {{ gateone_bin }}
nginx-location = {{ nginx_location}}
template-ngnix-conf = {{ template_nginx_conf }}
[dynamic-template-gateone]
< = jinja2-template-base
template = {{ template_gateone }}
filename = instance-gateone.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-gateone-parameters
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-template-gateone:rendered}
gateone = ${:default}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/software.cfg 0000664 0000000 0000000 00000005531 13712426147 0027154 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../component/dash/buildout.cfg
../../component/git/buildout.cfg
../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg
../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/kerberos/buildout.cfg
../../component/python-kerberos/buildout.cfg
../../component/gateone/buildout.cfg
../../component/dtach/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/slapos.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
kerberos
eggs
gateone-develop
gateone
dash
check-recipe
template
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
${python-cryptography:egg}
${python-kerberos:egg}
tornado
scripts =
slapos-kill
[extra-eggs]
recipe = zc.recipe.egg
interpreter = python
eggs =
${lxml-python:egg}
${python-kerberos:egg}
tornado
gateone
setuptools
pyOpenSSL
futures
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.git
branch = re6st-master
git-executable = ${git:location}/bin/git
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
mode = 644
[template-jinja2-base]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}.in
rendered = ${buildout:directory}/${:filename}
# XXX: extra-context is needed because we cannot append to a key of an extended
# section.
extra-context =
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
${:extra-context}
[template]
< = template-jinja2-base
filename = template.cfg
template = ${:_profile_base_location_}/${:_update_hash_filename_}
extra-context =
key dash_location dash:location
key dtach_location dtach:location
key gateone_location gateone-repository:location
key logrotate_location logrotate:location
key nginx_location nginx:location
key openssl_location openssl:location
key readline_location readline:location
key template_nginx_conf template-nginx-conf:target
key template_logrotate_base template-logrotate-base:rendered
key template_gateone template-gateone:target
raw gateone_bin ${buildout:bin-directory}/gateone
[template-gateone]
< = download-base
[template-logrotate-base]
< = template-jinja2-base
filename = instance-logrotate-base.cfg
extra-context =
key dcron_location dcron:location
key gzip_location gzip:location
key logrotate_location logrotate:location
[template-nginx-conf]
< = download-base
url = ${:_profile_base_location_}/templates/${:filename}.in
filename = nginx.conf
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/gateone.egg-link
[versions]
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/templates/ 0000775 0000000 0000000 00000000000 13712426147 0026633 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gateone/templates/nginx.conf.in 0000664 0000000 0000000 00000003400 13712426147 0031227 0 ustar 00root root 0000000 0000000 worker_processes {{ parameter_dict['nb_workers'] }};
pid {{ parameter_dict['pid'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
error_page 401 /login;
location / {
proxy_pass_header Server;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Scheme $scheme;
proxy_pass {{ parameter_dict['backend-url'] }};
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/ 0000775 0000000 0000000 00000000000 13712426147 0024455 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/buildout.hash.cfg 0000664 0000000 0000000 00000006055 13712426147 0027715 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = dc3f318e8a3aa7a59f9394118543e9e3
[watcher]
_update_hash_filename_ = watcher.in
md5sum = 90690e1351637f20ff2df57a6c3e85b4
[gitlab-export]
_update_hash_filename_ = gitlab-export.in
md5sum = a7b32680e80f34276f0a32a5e22dad50
[database.yml.in]
_update_hash_filename_ = template/database.yml.in
md5sum = 61d1d04b9347b3168a1ad7676e4681ef
[gitconfig.in]
_update_hash_filename_ = template/gitconfig.in
md5sum = eb1230fee50067924ba89f4dc6e82fa9
[gitlab-parameters.cfg]
_update_hash_filename_ = gitlab-parameters.cfg
md5sum = c2e23c0f7baa1633df0436ca4e728424
[gitlab-shell-config.yml.in]
_update_hash_filename_ = template/gitlab-shell-config.yml.in
md5sum = 52d18b521b8cd16352fc88b1e1d79d53
[gitlab-unicorn-startup.in]
_update_hash_filename_ = gitlab-unicorn-startup.in
md5sum = aff91edaf9786c213db8ea703ab3571e
[gitlab.yml.in]
_update_hash_filename_ = template/gitlab.yml.in
md5sum = f4cc0bc898b8d59010d61473e2adc53b
[gitaly-config.toml.in]
_update_hash_filename_ = template/gitaly-config.toml.in
md5sum = 0f1ec4077dab586cc003ae13f689eda2
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = dfeff229aa696bb6b6051a2aff4301fe
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
md5sum = 2af7dcf63f74e5edc53a3ff11fa4989b
[instance-gitlab-test.cfg.in]
_update_hash_filename_ = instance-gitlab-test.cfg.in
md5sum = 7ba08928e6a8998ec8ed1bb97851b726
[macrolib.cfg.in]
_update_hash_filename_ = macrolib.cfg.in
md5sum = a56a44e96f65f5ed20211bb6a54279f4
[nginx-gitlab-http.conf.in]
_update_hash_filename_ = template/nginx-gitlab-http.conf.in
md5sum = cd7471a8c5d6f6bc848c62ce62dca966
[nginx.conf.in]
_update_hash_filename_ = template/nginx.conf.in
md5sum = 8c904510eb39dc212204f68f2b81b068
[rack_attack.rb.in]
_update_hash_filename_ = template/rack_attack.rb.in
md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
[resque.yml.in]
_update_hash_filename_ = template/resque.yml.in
md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in]
_update_hash_filename_ = template/smtp_settings.rb.in
md5sum = 4e1ced687a86e4cfff2dde91237e3942
[template-gitlab-resiliency-restore.sh.in]
_update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
md5sum = 16b9f52f00d55feab7e31a88029ad351
[unicorn.rb.in]
_update_hash_filename_ = template/unicorn.rb.in
md5sum = 67728235a2c4c9425c80f0c856749885
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/gitlab-export.in 0000664 0000000 0000000 00000001204 13712426147 0027563 0 ustar 00root root 0000000 0000000 #!{{ bash.location }}/bin/bash -e
# export gitlab site via `gitlab-backup pull` to a "for-export" git-backup repository
# gitlab-export
#
# NOTE gitlab-backup, gitlab-rails, ... all have to be on $PATH.
# which site to export is determined by which gitlab-rails is on $PATH.
if [ "$#" -ne 1 ]; then
echo "Usage: gitlab-export " 1>&2
exit 1
fi
exportto_repo="$1"
# create / setup export repository if it does not exist yet
mkdir -p "$exportto_repo"
cd "$exportto_repo"
# verify we are in a git repository
if ! git rev-parse --is-inside-git-dir ; then
git init --bare
fi
exec gitlab-backup pull
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/gitlab-parameters.cfg 0000664 0000000 0000000 00000011535 13712426147 0030546 0 ustar 00root root 0000000 0000000 # Upstream parameters for a GitLab instance
#
# Selected parameters - main ones - names and advanced defaults taken from omnibus-gitlab
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
#
# TODO better autogenerate from ^^^ (?)
#
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
[gitlab-parameters]
configuration.external_url = http://lab.example.com
# db advanced
configuration.db_pool = 10
# rack-attack
configuration.rate_limit_requests_per_period = 10
configuration.rate_limit_period = 60
configuration.time_zone = UTC
configuration.email_enabled = true
configuration.email_from = lab@example.com
configuration.email_display_name = GitLab
configuration.email_reply_to = noreply@example.com
configuration.smtp_enable = true
configuration.smtp_address = smtp.server
configuration.smtp_port = 465
configuration.smtp_user_name = smtp user
configuration.smtp_password = smtp password
configuration.smtp_domain = lab.example.com
configuration.smtp_authentication = login
configuration.smtp_enable_starttls_auto = true
# none | peer | client_once | fail_if_no_peer_cert -> see gitlab-omnibus links at top
configuration.smtp_openssl_verify_mode = peer
configuration.default_can_create_group = true
configuration.username_changing_enabled = true
configuration.default_theme = 2
configuration.default_projects_features.issues = true
configuration.default_projects_features.merge_requests = true
configuration.default_projects_features.wiki = true
configuration.default_projects_features.snippets = true
configuration.default_projects_features.builds = true
configuration.webhook_timeout = 10
# 0 means forever (seconds)
configuration.backup_keep_time = 0
# NOTE empty = default gitlab limits
configuration.git_max_size =
configuration.git_timeout =
# sidekiq
configuration.sidekiq_shutdown_timeout = 4
configuration.sidekiq_concurrency = 25
configuration.sidekiq_memory_killer_max_rss = 1000000
# unicorn
configuration.unicorn_worker_timeout = 60
configuration.unicorn_worker_processes = 2
# unicorn advanced
configuration.unicorn_backlog_socket = 1024
configuration.unicorn_worker_memory_limit_min = 300*(1024**2)
configuration.unicorn_worker_memory_limit_max = 350*(1024**2)
# nginx
configuration.nginx_client_max_body_size = 0
# NOTE: we don't really need old ciphers - usually we talk directly to frontend only
configuration.nginx_ssl_ciphers = ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4
configuration.nginx_ssl_prefer_server_ciphers = on
configuration.nginx_ssl_protocols = TLSv1 TLSv1.1 TLSv1.2
# the following is gitlab-omnibus default but not nginx's default
configuration.nginx_ssl_session_cache = builtin:1000 shared:SSL:10m
configuration.nginx_ssl_session_timeout = 5m
configuration.nginx_proxy_read_timeout = 300
configuration.nginx_proxy_connect_timeout = 300
# nginx advanced
configuration.nginx_worker_processes = 4
configuration.nginx_worker_connections = 10240
configuration.nginx_log_format = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
configuration.nginx_sendfile = on
configuration.nginx_tcp_nopush = on
configuration.nginx_tcp_nodelay = on
configuration.nginx_gzip = on
configuration.nginx_gzip_http_version = 1.0
configuration.nginx_gzip_comp_level = 2
configuration.nginx_gzip_proxied = any
configuration.nginx_gzip_types = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
configuration.nginx_keepalive_timeout = 65
configuration.nginx_header_allow_origin = $http_origin
configuration.nginx_hsts_max_age = 31536000
configuration.nginx_hsts_include_subdomains = false
configuration.nginx_gzip_enabled = true
# configuring trusted proxies
# GitLab is behind a reverse proxy, so we don't want the IP address of the proxy
# to show up as the client address (because rack attack blacklists the lab
# frontend)
configuration.nginx_real_ip_trusted_addresses =
configuration.nginx_real_ip_header = X-Forwarded-For
configuration.nginx_real_ip_recursive = off
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/gitlab-unicorn-startup.in 0000664 0000000 0000000 00000004767 13712426147 0031440 0 ustar 00root root 0000000 0000000 #!{{ bash_bin }}
# start up gitlab's unicorn with first making sure db is properly setup and all
# migrations are up as pre-condition.
RAKE={{ gitlab_rake }}
die() {
echo "$*" 1>&2
exit 1
}
# run psql on gitlab db
psql() {
{{ psql_bin }} \
-h {{ pgsql['pgdata-directory'] }} \
-U {{ pgsql.superuser }} \
-d {{ pgsql.dbname }} \
"$@"
}
# 1. what to do when instance is initially setup
# see
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/postgresql.rb
# initial db setup
# ( first quering PG several times waiting a bit till postgresql is started and ready )
tpgwait=5
while true; do
pgtables="$(psql -c '\d' 2>&1)" && break
tpgwait=$(( $tpgwait - 1 ))
test $tpgwait = 0 && die "pg query problem"
echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
sleep 1
done
echo "I: PostgreSQL ready." 1>&2
# make sure pg_trgm extension is enabled for gitlab db
psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' || die "pg_trgm setup failed"
if echo "$pgtables" | grep -q '^Did not find any relations' ; then
$RAKE db:schema:load db:seed_fu || die "initial db setup failed"
fi
# re-build ssh keys
# (we do not use them - just for cleannes)
force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed"
# 2. what to do when instance is upgraded
# see
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/upgrader.rb
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-ctl-commands/upgrade.rb
#
# Assets compilation is handled at instance deployment time. We do everything else here.
# make sure all migrations are up
migrate_log="{{ log_dir }}/db-migrate-`date +%s`.log"
$RAKE db:migrate >$migrate_log 2>&1 || die "db:migrate failed"
# if it was a no-op "migration" - we don't need info about that - only keep
# logs of actual migration run.
test -s $migrate_log || rm $migrate_log
# clear cache
$RAKE cache:clear || die "cache:clear failed"
# 3. finally exec to unicorn
exec {{ gitlab_unicorn }} \
-E production \
-c {{ unicorn_rb.rendered }} \
{{ gitlab_work.location }}/config.ru
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/gowork.cfg 0000664 0000000 0000000 00000002055 13712426147 0026450 0 ustar 00root root 0000000 0000000 # Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_pkg_errors:recipe}
${go_lab.nexedi.com_kirr_git-backup:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
[go_github.com_libgit2_git2go]
<= go-git-package
go.importpath = github.com/libgit2/git2go
repository = https://github.com/libgit2/git2go.git
# branch 'next' is required by git-backup
revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors.git
revision = v0.8.0-12-g816c908556
[go_lab.nexedi.com_kirr_git-backup]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/git-backup
repository = https://lab.nexedi.com/kirr/git-backup.git
revision = 3f6c4deec8834bdcd2c28c7c5eeacd8211e759b5
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 56bf8f815a
instance-gitlab-export.cfg.in 0000664 0000000 0000000 00000006227 13712426147 0032056 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab # GitLab instance + site export support
[buildout]
extends = {{ instance_gitlab_cfg }}
# TODO + ${pbsready-export:output}
parts +=
cron-entry-gitlab-backup
resiliency-exclude-file
gitlab-resiliency-restore-script
# -export specific instance parameters
[instance-parameter]
# cron frequency for gitlab backup (default: every 4h)
configuration.backup_frequency = 0 */4 * * *
[gitlab-backup-directory]
recipe = slapos.cookbook:mkdirectory
srv = ${buildout:directory}/srv
backup = ${:srv}/backup
backup-gitlab.git = ${:backup}/backup-gitlab.git
var = ${buildout:directory}/var
pid = ${:var}/pid
# instance exporter script
[exporter]
recipe = slapos.cookbook:wrapper
wrapper-path = ${buildout:directory}/bin/gitlab-exporter
command-line = {{ xnice_repository_location }}/bin/xnice {{ gitlab_export }} ${gitlab-backup-directory:backup-gitlab.git}
pidfile = ${gitlab-backup-directory:pid}/gitlab-exporter.pid
environment =
# XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
# `which` command in gitlab-backup, `chrt` in xnice, ...
# and `/bin` for `sed` command in gitlab-backup restore
PATH=${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin:/bin:/usr/bin
[cron-entry-gitlab-backup]
<= cron-entry
# run backup script on a regular basis (given as instance parameter)
frequency = ${instance-parameter:configuration.backup_frequency}
command = ${exporter:wrapper-path}
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: gitlab-shell-work*
gitlab-work*
var/log/**
var/backup/**
var/repositories*
var/repositories/**
srv/postgresql/**
srv/postgresql
etc/service/postgres-start
srv/redis/**
srv/unicorn/unicorn.socket
output = ${directory:srv}/exporter.exclude
[gitlab-resiliency-restore-script]
# script run by resilient stack to restore gitlab instance
# this section should be added only on runner import instance
recipe = slapos.recipe.template:jinja2
mode = 0700
template= {{ gitlab_restore_sh_in }}
rendered= ${directory:srv}/runner-import-restore
context =
raw bash_bin {{ bash_bin }}
raw go_work_bin {{ gopath_bin }}
raw git_location {{ git_location }}
raw bin_directory ${directory:bin}
raw etc_directory ${directory:etc}
raw run_directory ${directory:run}
raw postgress_script ${service-postgresql:services}/postgres-start
raw redis_script ${service-redis:wrapper}
raw unicorn_script ${service-unicorn:wrapper-path}
raw sidekiq_script ${service-sidekiq:wrapper-path}
raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git}
raw redis_pid_file ${service-redis:pid_file}
raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid
raw gitlab_work_location ${gitlab-work:location}
raw promise_lab_location ${directory:promise.slow}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/instance-gitlab-test.cfg.in0000664 0000000 0000000 00000003203 13712426147 0031562 0 ustar 00root root 0000000 0000000 [buildout]
extends =
{{ instance_gitlab_export_cfg }}
parts +=
install-demo-backup
[root-password]
passwd = root1234
[service-unicorn]
environment =
GITLAB_ROOT_PASSWORD=${root-password:passwd}
[service-postgresql]
pgdata-directory = ${directory:srv}/pg
[gitlab-workhorse-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/glab-wh
[gitlab-workhorse]
srv = ${gitlab-workhorse-dir:srv}
socket = ${gitlab-workhorse:srv}/wh.socket
[unicorn]
socket = ${:srv}/unc.socket
[publish-instance-info]
password = ${root-password:passwd}
# token for default.user user in gitlab demo backup
# Edit this token if needed
private-token = SLurtnxPscPsU-SDm4oN
# raw URL for latest commit on setup.py in gitlab demo backup.
latest-file-uri = ${:backend_url}/open/slapos/raw/94c96d42c22e16836dadddac7c8061f4a8c6ca7a/setup.py
[instance-parameter]
# backup more often, 10 minutes seems the minimal
configuration.backup_frequency = */10 * * * *
[install-demo-backup]
recipe = plone.recipe.command
stop-on-error = false
backup-done = ${directory:var}/backup.ready
command =
if [ -f "${:backup-done}" ]; then
echo "Demo backup installed."
else
rm -rf ${secrets:secrets} ${directory:var}/backup/* &&
mkdir -p ${secrets:secrets} ${directory:var}/tmp &&
cp -r {{ gitlab_demo_backup_path }}/secrets/* ${secrets:secrets} &&
cp -rf {{ gitlab_demo_backup_path }}/backup-gitlab.git/ ${directory:var}/tmp &&
cd ${directory:var}/tmp/backup-gitlab.git/
PATH=${directory:bin}:{{ gopath_bin }}:{{ git_location }}/bin:$PATH
gitlab-backup restore -vupok -go HEAD &&
touch ${:backup-done}
fi
update-command = ${:command}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/instance-gitlab.cfg.in 0000664 0000000 0000000 00000061310 13712426147 0030610 0 ustar 00root root 0000000 0000000 # GitLab instance
# NOTE instance/software layout is inspired by gitlab omnibus
# NOTE all services are interconnected via unix sockets - because of easier
# security and performance reasons (unix has 2x less latency and more
# throughput compared to tcp over loopback).
[buildout]
extends =
{{ gitlab_parameters_cfg }}
{{ monitor_template }}
parts =
directory
publish-instance-info
# gitlab-
# ? mailroom
{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %}
{% for prog in gitlab_progv %}
gitlab-{{ prog }}
{% endfor %}
gitconfig
gitlab-work
gitlab-shell-work
service-gitlab-workhorse
service-unicorn
service-sidekiq
service-nginx
service-postgresql
service-redis
promise-redis
service-gitaly
cron-service
cron-entry-logrotate
logrotate-entry-cron
on-reinstantiate
# std stuff for slapos instance
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
##################################
# GitLab instance parameters #
##################################
[instance-parameter]
# std stuff to fetch slapos instance parameters
recipe = slapos.cookbook:slapconfiguration
computer= ${slap-connection:computer-id}
partition=${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# autogenerated gitlab instance parameters
<= gitlab-parameters
# adjust/override some default settings:
# automatically load all available CPUs
configuration.unicorn_worker_processes = {{ multiprocessing.cpu_count() + 1 }}
configuration.nginx_worker_processes = {{ multiprocessing.cpu_count() }}
# gitlab non-native parameters
configuration.icp_license =
# for convenience
[external-url]
recipe = slapos.cookbook:urlparse
url = ${instance-parameter:configuration.external_url}
[backend-info]
host = ${instance-parameter:ipv6-random}
port = 7777
# whether to use http or https - determined by external url
url = ${external-url:scheme}://[${:host}]:${:port}
# current slapuserX
user = {{ pwd.getpwuid(os.getuid())[0] }}
[publish-instance-info]
recipe = slapos.cookbook:publish
backend_url = ${backend-info:url}
#############################
# GitLab instance setup #
#############################
# 1. directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
etc = ${:home}/etc
var = ${:home}/var
log = ${:var}/log
run = ${:var}/run
srv = ${:home}/srv
# slapos startup/service/promise scripts live here:
startup = ${:etc}/run
service = ${:etc}/service
promise.slow = ${:etc}/promise.slow
# gitlab: etc/ log/ ...
[gitlab-dir]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/gitlab
log = ${directory:log}/gitlab
var = ${directory:var}/gitlab
tmp = ${:var}/tmp
uploads = ${:var}/uploads
assets = ${:var}/assets
shared = ${:var}/shared
artifacts = ${:shared}/artifacts
lfs-objects = ${:shared}/lfs-objects
builds = ${:var}/builds
backup = ${directory:var}/backup
public = ${:var}/public
pages = ${:shared}/pages
[gitlab-repo-dir]
recipe = slapos.cookbook:mkdirectory
repositories = ${directory:var}/repositories
# gitlab wants it to be drwxrws---
# FIXME setting such mode with :mkdirectory is not possible, because mkdir(2)
# does & 0777 and also there is umask. So we workaround:
[gitlab-repo-xdir]
recipe = plone.recipe.command
stop-on-error = yes
repositories = ${gitlab-repo-dir:repositories}
command = chmod 02770 ${:repositories}
[gitlab]
etc = ${gitlab-dir:etc}
log = ${gitlab-dir:log}
var = ${gitlab-dir:var}
tmp = ${gitlab-dir:tmp}
uploads = ${gitlab-dir:uploads}
assets = ${gitlab-dir:assets}
shared = ${gitlab-dir:shared}
artifacts = ${gitlab-dir:artifacts}
lfs-objects = ${gitlab-dir:lfs-objects}
builds = ${gitlab-dir:builds}
backup = ${gitlab-dir:backup}
repositories = ${gitlab-repo-xdir:repositories}
public = ${gitlab-dir:public}
pages = ${gitlab-dir:shared}/pages
# gitlab-shell: etc/ log/ gitlab_shell_secret ...
[gitlab-shell-dir]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/gitlab-shell
log = ${directory:log}/gitlab-shell
[gitlab-shell]
etc = ${gitlab-shell-dir:etc}
log = ${gitlab-shell-dir:log}
secret = ${secrets:secrets}/gitlab_shell_secret
hook =
# place to keep all secrets
[secrets]
recipe = slapos.cookbook:mkdirectory
secrets = ${directory:var}/secrets
mode = 0700
[gitaly-dir]
recipe = slapos.cookbook:mkdirectory
gitaly = ${directory:var}/gitaly
sockets = ${:gitaly}/sockets
internal = ${:sockets}/internal
log = ${directory:log}/gitaly
[gitaly]
socket = ${gitaly-dir:sockets}/gitaly.socket
log = ${gitaly-dir:log}
location = {{ gitaly_location }}
pid = ${directory:run}/gitaly.pid
internal_socket = ${gitaly-dir:internal}
# 2. configuration files
[etc-template]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
mode = 0640
import-list =
rawfile macrolib.cfg.in {{ macrolib_cfg_in }}
context =
raw autogenerated # This file was autogenerated. (DO NOT EDIT - changes will be lost)
section instance_parameter instance-parameter
section backend_info backend-info
import urlparse urlparse
raw git {{ git }}
${:context-extra}
context-extra =
[gitlab-etc-template]
<= etc-template
rendered= ${gitlab:etc}/${:_buildout_section_name_}
[nginx-etc-template]
<= etc-template
rendered= ${nginx:etc}/${:_buildout_section_name_}
[database.yml]
<= gitlab-etc-template
template= {{ database_yml_in }}
context-extra =
section pgsql service-postgresql
[gitconfig]
<= etc-template
template= {{ gitconfig_in }}
# NOTE put directly into $HOME/ - this way git will pick it up
rendered= ${directory:home}/.${:_buildout_section_name_}
[gitlab-shell-config.yml]
<= etc-template
template= {{ gitlab_shell_config_yml_in }}
rendered= ${gitlab-shell:etc}/config.yml
context-extra =
import urllib urllib
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section unicorn unicorn
section service_redis service-redis
raw redis_binprefix {{ redis_binprefix }}
[gitlab.yml]
<= gitlab-etc-template
template= {{ gitlab_yml_in }}
context-extra =
import urllib urllib
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section gitaly gitaly
[nginx.conf]
<= nginx-etc-template
template= {{ nginx_conf_in }}
context-extra =
section directory directory
section gitlab_workhorse gitlab-workhorse
raw nginx_mime_types {{ nginx_mime_types }}
raw nginx_gitlab_http_conf ${nginx-gitlab-http.conf:rendered}
[nginx-gitlab-http.conf]
<= nginx-etc-template
template= {{ nginx_gitlab_http_conf_in }}
context-extra =
section nginx nginx
section gitlab_work gitlab-work
section gitlab_workhorse gitlab-workhorse
[gitaly-config.toml]
<= etc-template
template= {{ gitaly_config_toml_in }}
rendered= ${directory:etc}/${:_buildout_section_name_}
context-extra =
import urllib urllib
section gitlab gitlab
section gitlab_shell_work gitlab-shell-work
section gitaly gitaly
[rack_attack.rb]
<= gitlab-etc-template
template = {{ rack_attack_rb_in }}
[resque.yml]
<= gitlab-etc-template
template= {{ resque_yml_in }}
context-extra =
section redis service-redis
[smtp_settings.rb]
<= gitlab-etc-template
template= {{ smtp_settings_rb_in }}
# contains smtp password
mode = 0600
[unicorn.rb]
<= gitlab-etc-template
template = {{ unicorn_rb_in }}
context-extra =
section unicorn unicorn
section directory directory
section gitlab_work gitlab-work
# 3. bin/
# gitlab-
[gitlab-bin]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
# NOTE $HOME needed to pick gitconfig
environment =
PATH = {{ node_bin_location }}:{{ gopath_bin }}:{{ yarn_location }}/bin:/usr/local/bin:/usr/bin:/bin
BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile
HOME = ${directory:home}
RAILS_ENV = production
SIDEKIQ_MEMORY_KILLER_MAX_RSS = ${instance-parameter:configuration.sidekiq_memory_killer_max_rss}
command-line =
{{ bundler_4gitlab }} exec sh -c
'cd ${gitlab-work:location} && ${:prog} "$@"' ${:prog}
{% for prog in gitlab_progv %}
[gitlab-{{ prog }}]
<= gitlab-bin
prog = {{ prog }}
{% endfor %}
[gitlab-unicorn-startup]
recipe = slapos.recipe.template:jinja2
mode = 0755
template= {{ gitlab_unicorn_startup_in }}
rendered= ${directory:bin}/${:_buildout_section_name_}
context =
raw bash_bin {{ bash_bin }}
raw gitlab_rake ${gitlab-rake:wrapper-path}
raw gitlab_unicorn ${gitlab-unicorn:wrapper-path}
raw psql_bin {{ postgresql_location }}/bin/psql
section pgsql service-postgresql
raw log_dir ${gitlab:log}
section unicorn_rb unicorn.rb
section gitlab_work gitlab-work
# 4. gitlab- & gitlab-shell- work directories
#
# Gitlab/Rails operation is tightened that config/ lives inside code, which goes
# against having ability to create several instances configured differently
# from 1 SR.
#
# One possibility to overcome this could be to make another Gitlab root
# symbolically linked to original SR _and_ several configuration files
# symbolically linked to instance place. Unfortunately this does not work -
# Ruby determines realpath on module import and Gitlab and Rails lookup config
# files relative to imported modules.
#
# we clone cloned gitlab and add proper links to vendor/bundle and instance
# config files.
# XXX there is no need for full clone - we only need worktree checkout (a-la `git
# worktree add`, but without creating files in original clone)
#
# This way Gitlab/Rails still think they work in 1 code / 1 instance way,
# and we can reuse SR.
# XXX better do such tricks with bind mounting, but that requires user namespaces
[work-base]
recipe = plone.recipe.command
stop-on-error = yes
location = ${directory:home}/${:_buildout_section_name_}
command =
# make sure we start from well-defined empty state
# (needed e.g. if previous install failed in the middle)
rm -rf ${:location} &&
# init work repository and add `software` remote pointing to main repo in SR software/...
{{ git }} init ${:location} &&
cd ${:location} &&
{{ git }} remote add software ${:software} &&
${:update-command}
update-command =
cd ${:location} &&
{{ git }} fetch software &&
{{ git }} reset --hard `cd ${:software} && {{ git }} rev-parse HEAD` &&
${:tune-command}
# NOTE there is no need to link/create .gitlab_shell_secret - we set path to it
# in gitlab & gitlab-shell configs, and gitlab creates it on its first start
[gitlab-work]
<= work-base
software = {{ gitlab_repository_location }}
tune-command =
# Initialise secrets
if [ ! -s "${secrets:secrets}/gitlab_secrets.yml" ]; then
cp config/secrets.yml.example ${secrets:secrets}/gitlab_secrets.yml;
fi
# secret* tmp/ log/ shared/ builds/ node_modules/
rm -f .secret &&
rm -rf log tmp shared builds node_modules &&
ln -sf ${secrets:secrets}/gitlab_rails_secret .secret &&
ln -sf ${gitlab:log} log &&
ln -sf ${gitlab:tmp} tmp &&
ln -sf ${gitlab:shared} shared &&
ln -sf ${gitlab:builds} builds &&
ln -sf {{ gitlab_repository_location }}/node_modules node_modules &&
ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
# config/
cd config &&
ln -sf ${unicorn.rb:rendered} unicorn.rb &&
ln -sf ${gitlab.yml:rendered} gitlab.yml &&
ln -sf ${database.yml:rendered} database.yml &&
ln -sf ${resque.yml:rendered} resque.yml &&
ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml &&
# config/initializers/
cd initializers &&
ln -sf ${rack_attack.rb:rendered} rack_attack.rb &&
ln -sf ${smtp_settings.rb:rendered} smtp_settings.rb &&
# public/
cd ../../public &&
rm -rf uploads assets &&
ln -sf ${gitlab:uploads} uploads &&
ln -sf ${gitlab:assets} assets &&
true
# ----//---- for gitlab-shell
[gitlab-shell-work]
<= work-base
software = {{ gitlab_shell_repository_location }}
tune-command =
ln -sf ${gitlab-shell-config.yml:rendered} config.yml &&
true
# 5. services
# [promise-] to check by url
[promise-byurl]
<= monitor-promise-base
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-http_code = 200
#####################
# Postgresql db #
#####################
# XXX gitlab-omnibus also tunes:
# - shared_buffers
# - work_mem
# - checkpoint_*
# - effective_check_size
# - lc_* en_US.UTF-8 -> C (?)
[service-postgresql]
recipe = slapos.cookbook:postgres
bin = {{ postgresql_location }}/bin
services= ${directory:service}
dbname = gitlabhq_production
# NOTE db name must match to what was used in KVM on lab.nexedi.com (restore script grants access to this user)
superuser = gitlab-psql
# no password - pgsql will listen only on unix sockets (see below) thus access
# is protected with filesystem-level permissions.
# ( besides, if we use slapos.cookbook:generate.password and do `password = ...`
# the password is stored in plain text in .installed and thus becomes insecure )
password=
pgdata-directory = ${directory:srv}/postgresql
# empty addresses - listen only on unix socket
ipv4 =
ipv6 =
port =
depend =
${promise-postgresql:recipe}
[promise-postgresql]
<= monitor-promise-base
module = check_command_execute
name = promise-postgresql.py
config-command =
{{ postgresql_location }}/bin/psql \
-h ${service-postgresql:pgdata-directory} \
-U ${service-postgresql:superuser} \
-d ${service-postgresql:dbname} \
-c '\q'
# postgresql logs to stdout/stderr - logs are handled by slapos not us
# [logrotate-entry-postgresql]
#############
# Redis #
#############
[redis]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/redis
log = ${directory:log}/redis
[service-redis]
recipe = slapos.cookbook:redis.server
wrapper = ${directory:service}/redis
promise_wrapper = ${directory:bin}/redis-promise
server_dir = ${redis:srv}
config_file = ${directory:etc}/redis.conf
log_file = ${redis:log}/redis.log
pid_file = ${directory:run}/redis.pid
use_passwd = false
unixsocket = ${:server_dir}/redis.socket
# port = 0 means "don't listen on TCP at all" - listen only on unix socket
ipv6 = ::1
port = 0
server_bin = {{ redis_binprefix }}/redis-server
depend =
${logrotate-entry-redis:recipe}
[promise-redis]
<= monitor-promise-base
module = check_command_execute
name = promise-redis.py
config-command = ${service-redis:promise_wrapper}
# NOTE slapos.cookbook:redis.server setups promise automatically
[logrotate-entry-redis]
<= logrotate-entry-base
log = ${redis:log}/*.log
name = redis
########################
# gitlab-workhorse #
########################
[gitlab-workhorse-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/gitlab-workhorse
log = ${directory:log}/workhorse
[gitlab-workhorse]
srv = ${gitlab-workhorse-dir:srv}
socket = ${gitlab-workhorse:srv}/gitlab-workhorse.socket
log = ${gitlab-workhorse-dir:log}/gitlab-workhorse.log
secret = ${secrets:secrets}/gitlab_workhorse_secret
[service-gitlab-workhorse]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/gitlab-workhorse
command-line = {{ gitlab_workhorse }}
-listenNetwork unix
-listenAddr ${gitlab-workhorse:socket}
-authSocket ${unicorn:socket}
-documentRoot ${gitlab-work:location}/public
-secretPath ${gitlab-workhorse:secret}
-logFile ${gitlab-workhorse:log}
# NOTE for profiling
# -pprofListenAddr ...
# NOTE environment for:
# - git to be available on path
# - ruby to be available on path (gitlab-workhorse -> gitlab-shell -> hooks on push)
# - gitconfig be found from ~/.gitconfig
environment =
PATH={{ git_location }}/bin:{{ ruby_location }}/bin:{{ gzip_location }}/bin:{{ bzip2_location}}/bin
HOME=${directory:home}
depend =
${promise-gitlab-workhorse:recipe}
${logrotate-entry-gitlab-workhorse:recipe}
[promise-gitlab-workhorse]
<= promise-byurl
# http://localhost/users/statics.css will not redirect to /users/sign_in anymore because of this commit:
# https://lab.nexedi.com/nexedi/gitlab-workhorse/commit/c81f109a62fecf2a847fb17ceed012b380dab49f#c1215002e6d745f05eaaf9ee1dad7752e85d866f_318_331
config-command = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket} http://localhost/users/sign_in
# gitlab-workhorse logs to stdout/stderr - logs are handled by slapos not us
# [logrotate-entry-gitlab-workhorse]
######################
# unicorn worker #
######################
[unicorn-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/unicorn
log = ${directory:log}/unicorn
[unicorn]
srv = ${unicorn-dir:srv}
log = ${unicorn-dir:log}
socket = ${:srv}/unicorn.socket
[service-unicorn]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/unicorn
# NOTE we perform db setup / migrations as part of unicorn startup.
# Those operations require PG and Redis to be up and running already, that's
# why we do it here. See gitlab-unicorn-startup for details.
command-line = ${gitlab-unicorn-startup:rendered}
depend =
${promise-unicorn:recipe}
${promise-gitlab-app:recipe}
${promise-gitlab-shell:recipe}
${logrotate-entry-unicorn:recipe}
# gitlab is a service "run" under unicorn
# gitlab-shell is called by gitlab
# -> associate their logs rotation to here
${logrotate-entry-gitlab:recipe}
[promise-unicorn]
<= promise-byurl
config-command = {{ curl_bin }} --unix-socket ${unicorn:socket} http://localhost/
[promise-rakebase]
recipe = slapos.cookbook:wrapper
# FIXME gitlab-rake is too slow to load and promise timeouts
# that's why we instantiate to .slow/ (and this way promises are not run)
wrapper-path = !py!'${directory:promise.slow}/' + '${:_buildout_section_name_}'[8:]
rake = ${gitlab-rake:wrapper-path}
[promise-gitlab-app]
<= promise-rakebase
command-line = ${:rake} gitlab:app:check
[promise-gitlab-shell]
<= promise-rakebase
command-line = ${:rake} gitlab:gitlab_shell:check
# very slow
# rake gitlab:repo:check (fsck all repos)
[logrotate-entry-unicorn]
<= logrotate-entry-base
log = ${unicorn:log}/*.log
name = unicorn
[logrotate-entry-gitlab]
<= logrotate-entry-base
log = ${gitlab:log}/*.log
name = gitlab
[logrotate-entry-gitlab-shell]
<= logrotate-entry-base
log = ${gitlab-shell:log}/*.log
name = gitlab-shell
[logrotate-entry-gitlab-workhorse]
<= logrotate-entry-base
log = ${gitlab-workhorse-dir:log}//*.log
name = gitlab-shell
#######################################
# sidekiq background jobs manager #
#######################################
[sidekiq-dir]
recipe = slapos.cookbook:mkdirectory
log = ${directory:log}/sidekiq
[sidekiq]
log = ${sidekiq-dir:log}
# NOTE see queue list here:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
[service-sidekiq]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/sidekiq
command-line =
# NOTE Sidekiq memory killer makes sidekiq processes to exit, or if exit request
# not handled in time, to be SIGKILL terminated, and relies on managing service
# to restart it. In slapos we don't have mechanism to set autorestart=true, nor
# bang/watchdog currently work with slapproxy, so we do the monitoring ourselves.
{{ watcher }} 0,SIGKILL
${gitlab-sidekiq:wrapper-path}
# XXX -q runner ? (present in gitlab-ce/Procfile but not in omnibus)
# XXX -q pages -q elasticsearch ? (present in omnibus but not in gitlab-ce -- those features are gitlab-ee only)
# XXX -P ? (pidfile)
-e production
-r ${gitlab-work:location}
-t ${instance-parameter:configuration.sidekiq_shutdown_timeout}
-c ${instance-parameter:configuration.sidekiq_concurrency}
-L ${sidekiq:log}/sidekiq.log
-C ${gitlab-work:location}/config/sidekiq_queues.yml
depend =
${promise-sidekiq:recipe}
${logrotate-entry-sidekiq:recipe}
[promise-sidekiq]
<= promise-rakebase
command-line = ${:rake} gitlab:sidekiq:check
[logrotate-entry-sidekiq]
<= logrotate-entry-base
log = ${sidekiq:log}/*.log
name = sidekiq
######################
# Nginx frontend #
######################
# srv/nginx/ prefix + etc/ log/ ...
[nginx-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/nginx
etc = ${directory:etc}/nginx
log = ${directory:log}/nginx
[nginx-ssl-dir]
recipe = slapos.cookbook:mkdirectory
ssl = ${nginx-dir:etc}/ssl
# contains https key
mode = 0700
# self-signed certificate for https
[nginx-generate-certificate]
# NOTE there is slapos.cookbook:certificate_authority.request but it requires
# to start whole service and has up to 60 seconds latency to generate
# certificate. We only need to run 1 command to do it...
recipe = plone.recipe.command
stop-on-error = true
cert_file = ${nginx-ssl-dir:ssl}/gitlab_backend.crt
key_file = ${nginx-ssl-dir:ssl}/gitlab_backend.key
command =
test -e ${:key_file} || \
{{ openssl_bin }} req -newkey rsa -batch -new -x509 -days 3650 -nodes \
-keyout ${:key_file} -out ${:cert_file}
update-command = ${:command}
[nginx]
srv = ${nginx-dir:srv}
etc = ${nginx-dir:etc}
log = ${nginx-dir:log}
ssl = ${nginx-ssl-dir:ssl}
cert_file = ${nginx-generate-certificate:cert_file}
key_file = ${nginx-generate-certificate:key_file}
[nginx-symlinks]
# (nginx wants /logs to be there from start - else it issues alarm to the log)
recipe = cns.recipe.symlink
symlink = ${nginx:log} = ${nginx:srv}/logs
[service-nginx]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/nginx
command-line = {{ nginx_bin }} -p ${nginx:srv} -c ${nginx.conf:rendered}
depend =
${nginx-symlinks:recipe}
${promise-nginx:recipe}
${logrotate-entry-nginx:recipe}
[promise-nginx]
<= promise-byurl
# XXX this depends on gitlab-workhorse being up
# (nginx is configured to proxy all requests to gitlab-workhorse)
config-url = ${backend-info:url}/users/sign_in
module = check_url_available
[logrotate-entry-nginx]
<= logrotate-entry-base
log = ${nginx:log}/*.log
name = nginx
# base entry for clients who registers to cron
[cron-entry]
recipe = slapos.cookbook:cron.d
# name = .strip_prefix('cron-entry-')
# XXX len() is not available in !py! - 11 hardcoded
name = !py!'${:_buildout_section_name_}' [11:]
# NOTE _not_ ${service-cron:cron-entries} - though the value is the same we do
# not want service-cron to be instantiated just if a cron-entry is registered.
cron-entries = ${cron:cron-entries}
######################
# gitaly worker #
######################
# https://docs.gitlab.com/ee/install/installation.html
[service-gitaly]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/gitaly
#command-line = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
command-line = {{ gitaly_location }}/gitaly ${gitaly-config.toml:rendered}
environment =
PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin
# 6. on-reinstantiate actions
# NOTE here we only recompile assets. Other on-reinstantiate actions, which
# require pg and redis running, are performed as part of unicorn service -
# right before its startup (see gitlab-unicorn-startup).
[on-reinstantiate]
recipe = plone.recipe.command
stop-on-error = true
rake = ${gitlab-rake:wrapper-path}
# run command on every reinstantiation
update-command = ${:command}
# https://gitlab.com/gitlab-org/gitlab-foss/issues/38457
# we need to manually install ajv@^4.0.0 with yarn to fix the bug 'yarn check failed!'
command =
${:rake} gitlab:assets:clean &&
${:rake} gettext:compile RAILS_ENV=production &&
cd ${gitlab-work:location} &&
PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn add ajv@^4.11.2 &&
PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn install --production --pure-lockfile &&
${:rake} gitlab:assets:compile NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096" &&
true
# Promise, gitlab can connect to gitaly:
# sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT]
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/instance.cfg.in 0000664 0000000 0000000 00000010735 13712426147 0027355 0 ustar 00root root 0000000 0000000 # GitLab "switch-softwaretype" instance
[buildout]
parts = switch-softwaretype
# std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
gitlab = $${instance-gitlab.cfg:rendered}
gitlab-export = $${instance-gitlab-export.cfg:rendered}
gitlab-test = $${instance-gitlab-test.cfg:rendered}
default = $${:gitlab}
# TODO -import, -pull-backup
# macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg]
recipe = slapos.recipe.template:jinja2
mode = 0644
rendered= $${buildout:directory}/$${:_buildout_section_name_}
context =
import os os
import pwd pwd
import multiprocessing multiprocessing
key bin_directory buildout:bin-directory
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
raw gitlab_repository_location ${gitlab-repository:location}
raw gitlab_shell_repository_location ${gitlab-shell-repository:location}
# program binaries
raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle}
raw bundler_1_17_3_dir ${bundler-4gitlab:bundle1.17.3}
raw coreutils_location ${coreutils:location}
raw curl_bin ${curl:location}/bin/curl
raw dcron_bin ${dcron-output:crond}
raw git ${git:location}/bin/git
raw git_location ${git:location}
raw gitaly_location ${gitaly-repository:location}
raw gitlab_export ${gitlab-export:rendered}
raw gitlab_workhorse ${gowork:bin}/gitlab-workhorse
raw gopath_bin ${gowork:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location}
raw gzip_bin ${gzip:location}/bin/gzip
raw gzip_location ${gzip:location}
raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate
raw nginx_bin ${nginx-output:nginx}
raw nginx_mime_types ${nginx-output:mime}
raw node_bin_location ${nodejs-8.12.0:location}/bin/
raw openssl_bin ${openssl-output:openssl}
raw postgresql_location ${postgresql10:location}
raw redis_binprefix ${redis28:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location}
raw watcher ${watcher:rendered}
raw xnice_repository_location ${xnice-repository:location}
raw yarn_location ${yarn:location}
# config files
raw database_yml_in ${database.yml.in:target}
raw gitconfig_in ${gitconfig.in:target}
raw gitlab_parameters_cfg ${gitlab-parameters.cfg:target}
raw monitor_template ${monitor2-template:rendered}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target}
raw gitlab_yml_in ${gitlab.yml.in:target}
raw gitaly_config_toml_in ${gitaly-config.toml.in:target}
raw macrolib_cfg_in ${macrolib.cfg.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target}
raw rack_attack_rb_in ${rack_attack.rb.in:target}
raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target}
$${:context-extra}
context-extra =
[instance-gitlab.cfg]
<= instance-cfg
template= ${instance-gitlab.cfg.in:target}
[instance-gitlab-export.cfg]
<= instance-cfg
template= ${instance-gitlab-export.cfg.in:target}
context-extra =
raw instance_gitlab_cfg $${instance-gitlab.cfg:rendered}
[instance-gitlab-test.cfg]
<= instance-cfg
template= ${instance-gitlab-test.cfg.in:target}
context-extra =
raw instance_gitlab_cfg $${instance-gitlab.cfg:rendered}
raw instance_gitlab_export_cfg $${instance-gitlab-export.cfg:rendered}
raw gitlab_demo_backup_path ${gitlab-demo-backup.git:location}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/macrolib.cfg.in 0000664 0000000 0000000 00000001431 13712426147 0027332 0 ustar 00root root 0000000 0000000 {# common macros for gitlab instance #}
{# cfg(name) -> instance_parameter:configuration. #}
{% macro cfg(name) %}{{ instance_parameter[str("configuration." + name)] }}{% endmacro %}
{# cfg_bool(name) - like cfg(name), but returns 'true'/''
NOTE macros can return only strings - that's why '' is used for false #}
{% macro cfg_bool(name) %}{{ 'true' if (cfg(name).lower() in ('true', 'yes')) else '' }}{% endmacro %}
{# deduce whether to use https from external url
( here - becasue we cannot use jinja2 logic in instance-gitlab.cfg.in to
process instance parameters ) #}
{% set external_url = urlparse.urlparse(cfg('external_url')) %}
{% set cfg_https = (true if external_url.scheme == 'https' else false) %}
{# for convenience #}
{% set fqdn = external_url.hostname %}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/software.cfg 0000664 0000000 0000000 00000027707 13712426147 0027005 0 ustar 00root root 0000000 0000000 # GitLab software-release
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/ruby/buildout.cfg
../../component/golang/buildout.cfg
../../component/postgresql/buildout.cfg
../../component/redis/buildout.cfg
../../component/cmake/buildout.cfg
../../component/icu/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../component/nodejs/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/zlib/buildout.cfg
gowork.cfg
# for instance
../../component/coreutils/buildout.cfg
../../component/bash/buildout.cfg
../../component/grep/buildout.cfg
../../component/bzip2/buildout.cfg
../../component/curl/buildout.cfg
../../component/tar/buildout.cfg
../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
parts =
ruby2.3
golang1.12
git
postgresql10
redis28
cmake
icu
pkgconfig
nginx-output
gowork
gitlab-workhorse
gitaly-build
python-4gitlab
gitlab-shell/vendor
gitlab/vendor/bundle
gitlab_npm
github-markup-patch
gitlab-backup
# for instance
instance.cfg
slapos-cookbook
eggs
bash
curl
watcher
gitlab-export
gzip
dcron-output
logrotate
[slapos.cookbook-repository]
revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
[yarn]
# need this version of Yarn
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
md5sum = db82fa09c996e9318f2f1d2ab99228f9
############################
# Software compilation #
############################
# python with eggs, that will be used in gitlab
[python-4gitlab]
recipe = zc.recipe.egg
interpreter = python2
eggs =
docutils
# rubygemsrecipe with fixed url and this way pinned rubygems version
[rubygemsrecipe]
recipe = rubygemsrecipe
url = https://rubygems.org/rubygems/rubygems-3.1.2.zip
# bundler, that we'll use to
# - install gems for gitlab
# - run gitlab services / jobs (via `bundle exec ...`)
[bundler-4gitlab]
<= rubygemsrecipe
ruby-location = ${ruby2.3:location}
ruby-executable = ${:ruby-location}/bin/ruby
gems =
bundler==1.17.3
# bin installed here
bundle = ${buildout:bin-directory}/bundle
# Gitaly need bundler 1.17.3 which is not the default version at the end
bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/gems/1.8/gems/bundler-1.17.3/exe/
# install together with dependencies of gitlab, which we cannot specify using
# --with-... gem option
# ( reason: rubygemsrecipe hardcodes PATH inside generated bin/* and it is
# impossible to adjust it later )
#
# bundle exec ; starts with `#!/usr/bin/env ruby` as rubygems
# Rugged needs: cmake, pkgconfig
# execjs needs: nodejs
# rails needs db client program on path: psql
# gitlab wants to check redis version via running: redis-cli
# gitlab wants git to be really on path ( it uses git from abspath defined in
# gitlab.yml, but there are not all cases like this, e.g. in
# https://gitlab.com/gitlab-org/gitlab_git/blob/2f0d3c1a/lib/gitlab_git/repository.rb#L259 )
# gitlab (via github-markup) wants to convert rst -> html via running: python2 (with docutils egg)
# (python-4gitlab puts interpreter into ${buildout:bin-directory})
environment =
PATH = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
# pinned to exact commit
[git-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
[gitlab-repository]
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 9.5.10 + NXD patches:
revision = v9.5.10-8-gc290e22a08cb
location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository]
<= git-repository
#repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
repository = https://gitlab.com/gitlab-org/gitlab-shell.git
# gitlab 9.5.10 wants gitlab-shell 5.6.1
revision = v5.6.1-10-g1e587d3b7f
location = ${buildout:parts-directory}/gitlab-shell
[gitaly-repository]
<= git-repository
repository = https://gitlab.com/gitlab-org/gitaly.git
# for version v0.35.0 (gitlab 9.5.10)
revision = v0.35.0-0-gf99a57b19a
location = ${buildout:parts-directory}/gitaly
[gitlab-workhorse-repository]
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
revision = v3.0.0-8-g74793ad3cc
# Patch github markup to not call "python2 -S /path/to/rest2html" but only "python2 /path/to/rest2html"
# NOTE github-markup invokes it as `python2`, that's why we are naming it this way
# https://github.com/github/markup/blob/5393ae93/lib/github/markups.rb#L36
[github-markup-patch]
recipe = plone.recipe.command
command =
files=$(ls ${gitlab-repository:location}/vendor/bundle/ruby/*/gems/git*-markup-*/lib/github/markups.rb) || true
if [ ! -z "$files" ]; then
for file in $files; do
sed -i 's#python2 -S#python2#' $file
done
fi
update-command = ${:command}
stop-on-error = True
# build needed-by-gitlab gems via bundler
[gitlab/vendor/bundle]
recipe = slapos.recipe.cmmi
path = ${gitlab-repository:location}
bundle = ${bundler-4gitlab:bundle}
configure-command = cd ${:path} &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} &&
${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
make-binary =
make-targets= cd ${:path} &&
${:bundle} install --deployment --without development test mysql aws kerberos
environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:%(PATH)s
CFLAGS=-I${xz-utils:location}/include
################## Google re2
[re2]
recipe = slapos.recipe.cmmi
url = https://github.com/google/re2/archive/2019-12-01.tar.gz
md5sum = 527eab0c75d6a1a0044c6eefd816b2fb
configure-command = :
[gitlab_npm]
recipe = slapos.recipe.cmmi
path = ${gitlab-repository:location}
configure-command = :
make-binary =
make-targets= cd ${:path} && npm install
environment =
PATH=${nodejs-8.12.0:location}/bin/:%(PATH)s
#our go infrastructure not currently supporting submodules, IIRC
# https://lab.nexedi.com/nexedi/slapos/merge_requests/337
[go_github.com_libgit2_git2go_prepare]
recipe = slapos.recipe.cmmi
path = ${go_github.com_libgit2_git2go:location}
configure-command = :
make-binary =
make-targets= cd ${go_github.com_libgit2_git2go:location}
&& git submodule update --init
&& sed -i 's/.*--build.*/cmake --build . --target install/' script/build-libgit2-static.sh
&& make install
environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.12:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory}
[gowork.goinstall]
git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install
command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
[gowork]
golang = ${golang1.12:location}
# gitlab.com/gitlab-org/gitlab-workhorse
# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
install =
lab.nexedi.com/kirr/git-backup
cpkgpath =
${openssl-1.0:location}/lib/pkgconfig
${zlib:location}/lib/pkgconfig
${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install/lib/pkgconfig
buildflags = --tags "static"
[gitlab-workhorse]
recipe = slapos.recipe.cmmi
path = ${gitlab-workhorse-repository:location}
md5sum = 2988c944d58c4a08880498c4981cc7b7
configure-command = :
make-binary =
make-targets =
. ${gowork:env.sh} && make install PREFIX=${gowork:directory}
[gitlab-backup]
recipe = plone.recipe.command
command =
cp -a ${go_lab.nexedi.com_kirr_git-backup:location}/contrib/gitlab-backup ${gowork:bin}
update-command = ${:command}
[gitaly-build]
recipe = slapos.recipe.cmmi
path = ${gitaly-repository:location}
bundle = ${bundler-4gitlab:bundle}
configure-command = cd ${:path}/ruby &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
make-binary =
make-targets =
. ${gowork:env.sh} && make
environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
[xnice-repository]
# to get kirr's misc repo containing xnice script for executing processes
# with lower priority (used for backup script inside the cron)
<= git-repository
repository = https://lab.nexedi.com/kirr/misc.git
revision = 4073572ea700bf1b115f3a135aebebe5b3b824e4
location = ${buildout:parts-directory}/misc
# build needed-by-gitlab-shell gems via bundler
# ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error
# on mkdir vendor/bundle, this part name is just /vendor )
[gitlab-shell/vendor]
recipe = slapos.recipe.cmmi
path = ${gitlab-shell-repository:location}
bundle = ${bundler-4gitlab:bundle}
configure-command = true
make-binary =
make-targets= cd ${:path} &&
${:bundle} install --deployment --without development test
###############################
# Trampoline for instance #
###############################
# eggs for instance.cfg
[eggs]
recipe = zc.recipe.egg
eggs =
plone.recipe.command
cns.recipe.symlink
collective.recipe.template
[instance.cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance.cfg
# macro: download a shell script and put it rendered into /bin/
[binsh]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_update_hash_filename_}
rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
context =
section bash bash
[watcher]
<= binsh
[gitlab-export]
<= binsh
# macro: download a file named in buildout.hash.cfg via _update_hash_filename_
#
# [filename]
# <= download-file
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[database.yml.in]
<= download-file
[gitconfig.in]
<= download-file
[gitlab-parameters.cfg]
<= download-file
[gitlab-shell-config.yml.in]
<= download-file
[gitlab-unicorn-startup.in]
<= download-file
[gitlab.yml.in]
<= download-file
[gitaly-config.toml.in]
<= download-file
[instance-gitlab.cfg.in]
<= download-file
[instance-gitlab-export.cfg.in]
<= download-file
[instance-gitlab-test.cfg.in]
<= download-file
[macrolib.cfg.in]
<= download-file
[nginx-gitlab-http.conf.in]
<= download-file
[nginx.conf.in]
<= download-file
[rack_attack.rb.in]
<= download-file
[resque.yml.in]
<= download-file
[smtp_settings.rb.in]
<= download-file
[template-gitlab-resiliency-restore.sh.in]
<= download-file
[unicorn.rb.in]
<= download-file
[gitlab-demo-backup.git]
recipe = hexagonit.recipe.download
url = https://lab.nexedi.com/alain.takoudjou/labdemo.backup/repository/archive.tar.gz?ref=master
md5sum = d40e5e211dc9a4e5ada9c0250377c639
strip-top-level-dir = true
[versions]
cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos002
slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/ 0000775 0000000 0000000 00000000000 13712426147 0026270 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/database.yml.in 0000664 0000000 0000000 00000001762 13712426147 0031172 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
# (last updated for 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg with context %}
production:
adapter: postgresql
encoding: unicode
{# collation is mainly for mysql
collation: <%= @db_collation %>
#}
database: {{ pgsql.dbname }}
pool: {{ cfg('db_pool') }}
{# XXX is it ok to use superuser, even if the whole database is only for gitlab? #}
username: '{{ pgsql.superuser }}'
{# we have no password - access is via unix socket #}
password:
host: '{{ pgsql["pgdata-directory"] }}'
port:
socket:
{# not needed for unix socket
sslmode: <%= single_quote(@db_sslmode) %>
sslrootcert: <%= single_quote(@db_sslrootcert) || single_quote(@db_sslca) %>
sslca: <%= single_quote(@db_sslca) || single_quote(@db_sslrootcert) %>
#}
gitaly-config.toml.in 0000664 0000000 0000000 00000002663 13712426147 0032256 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template # Example Gitaly configuration file
# Documentation lives at https://docs.gitlab.com/ee/administration/gitaly/ and
# https://docs.gitlab.com/ee//administration/gitaly/reference
socket_path = "{{ gitaly.socket }}"
# The directory where Gitaly's executables are stored
bin_dir = "{{ gitaly.location }}"
# # Optional: listen on a TCP socket. This is insecure (no authentication)
# listen_addr = "localhost:9999"
# tls_listen_addr = "localhost:8888
# # Optional: export metrics via Prometheus
# prometheus_listen_addr = "localhost:9236"
# # Git settings
[git]
bin_path = "{{ git }}"
[[storage]]
name = "default"
path = "{{ gitlab.repositories }}"
# # You can optionally configure more storages for this Gitaly instance to serve up
#
# [[storage]]
# name = "other_storage"
# path = "/mnt/other_storage/repositories"
#
# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
# [logging]
# format = "json"
# # Additionally exceptions can be reported to Sentry
# sentry_dsn = "https://:@sentry.io/
# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
# [prometheus]
# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
[gitaly-ruby]
# The directory where gitaly-ruby is installed
dir = "{{ gitaly.location }}/ruby"
[gitlab-shell]
# The directory where gitlab-shell is installed
dir = "{{ gitlab_shell_work.location }}"
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/gitconfig.in 0000664 0000000 0000000 00000001416 13712426147 0030573 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# global git configuration for GitLab
# see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
#
{% from 'macrolib.cfg.in' import cfg with context %}
# don't waste memory when packing (each thread uses own work memory)
# besides it packs better with 1 thread
[pack]
threads = 1
# don't allow corrupt/broken objects to go in
[receive]
fsckObjects = true
[user]
name = {{ cfg('email_display_name') }}
email = {{ cfg('email_from') }}
[core]
autocrlf = input
[gc]
auto = 0
gitlab-shell-config.yml.in 0000664 0000000 0000000 00000004761 13712426147 0033163 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
# GitLab user. git by default
user: {{ backend_info.user }}
# Url to gitlab instance. Used for api calls. Should end with a slash.
gitlab_url: "http+unix://{{ urllib.quote_plus(unicorn.socket) }}/"
http_settings:
{# we don't need any
<%= @http_settings.to_json if @http_settings %>
#}
# user: someone
# password: somepass
# ca_file: /etc/ssl/cert.pem
# ca_path: /etc/pki/tls/certs
# self_signed_cert: false
# Repositories path
# Give the canonicalized absolute pathname,
# REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!!
# Check twice that none of the components is a symlink, including "/home".
# repos_path: "{{ gitlab.repositories }}"
# File used as authorized_keys for gitlab user
# NOTE not used in slapos version (all access via https only)
auth_file: "{{ gitlab.var }}/sshkeys-notused"
# File that contains the secret key for verifying access to GitLab.
# Default is .gitlab_shell_secret in the root directory.
secret_file: "{{ gitlab_shell.secret }}"
# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
# Default is hooks in the gitlab-shell directory.
custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
# Redis settings used for pushing commit notices to gitlab
redis:
bin: {{ redis_binprefix }}/redis-cli
host: {# <%= @redis_host %> #}
port: {# <%= @redis_port %> #}
socket: {{ service_redis.unixsocket }}
database: {# <%= @redis_database %> #}
namespace: resque:gitlab
# Log file.
# Default is gitlab-shell.log in the root directory.
log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
# Log level. INFO by default
log_level:
# Audit usernames.
# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
# incurs an extra API call on every gitlab-shell command.
audit_usernames:
# Enable git-annex support
# git-annex allows managing files with git, without checking the file contents into git
# See https://git-annex.branchable.com/ for documentation
# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
# For CentOS: sudo yum install epel-release && sudo yum install git-annex
git_annex_enabled:
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/gitlab.yml.in 0000664 0000000 0000000 00000053036 13712426147 0030671 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %}
production: &base
#
# 1. GitLab app settings
# ==========================
## GitLab settings
gitlab:
## Web server settings (note: host is the FQDN, do not include http://)
{% set default_port = {'http': 80, 'https': 443} %}
host: {{ external_url.hostname }}
port: {{ external_url.port or default_port[external_url.scheme] }}
https: {{ cfg_https }}
{# ssh is disabled completely in slapos version
# Uncommment this line below if your ssh host is different from HTTP/HTTPS one
# (you'd obviously need to replace ssh.host_example.com with your own host).
# Otherwise, ssh host will be set to the `host:` value above
ssh_host: <%= @gitlab_ssh_host %>
#}
# WARNING: See config/application.rb under "Relative url support" for the list of
# other files that need to be changed for relative url support
{# we do not support relative URL
relative_url_root: <%= @gitlab_relative_url %>
#}
# Content Security Policy
# See https://guides.rubyonrails.org/security.html#content-security-policy
content_security_policy:
enabled: true
report_only: false
directives:
base_uri:
child_src:
connect_src: "'self' http://localhost:* ws://localhost:* wss://localhost:*"
default_src: "'self'"
font_src:
form_action:
frame_ancestors: "'self'"
frame_src: "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com"
img_src: "* data: blob:"
manifest_src:
media_src:
object_src: "'none'"
script_src: "'self' 'unsafe-eval' http://localhost:* https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://www.gstatic.com/recaptcha/ https://apis.google.com"
style_src: "'self' 'unsafe-inline'"
worker_src: "'self' blob:"
report_uri:
# Trusted Proxies
# Customize if you have GitLab behind a reverse proxy which is running on a different machine.
# Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
trusted_proxies:
{% for proxy in cfg("nginx_real_ip_trusted_addresses").split() %}
- {{ proxy }}
{% endfor %}
# Uncomment and customize if you can't use the default user to run GitLab (default: 'git')
user: {{ backend_info.user }}
## Date & Time settings
time_zone: '{{ cfg("time_zone") }}'
## Email settings
# Uncomment and set to false if you need to disable email sending from GitLab (default: true)
email_enabled: {{ cfg('email_enabled') }}
# Email address used in the "From" field in mails sent by GitLab
email_from: {{ cfg('email_from') }}
email_display_name: {{ cfg('email_display_name') }}
email_reply_to: {{ cfg('email_reply_to') }}
email_subject_suffix: ''
# Email server smtp settings are in [a separate file](initializers/smtp_settings.rb.sample).
## User settings
default_can_create_group: {{ cfg('default_can_create_group') }} # default: true
username_changing_enabled: {{ cfg('username_changing_enabled') }} # default: true - User can change her username/namespace
## Default theme
## 1 - Graphite
## 2 - Charcoal
## 3 - Green
## 4 - Gray
## 5 - Violet
## 6 - Blue
default_theme: {{ cfg('default_theme') }} # default: 2
{# for now we are ok with default issue-closing pattern
## Automatic issue closing
# If a commit message matches this regular expression, all issues referenced from the matched text will be closed.
# This happens when the commit is pushed or merged into the default branch of a project.
# When not specified the default issue_closing_pattern as specified below will be used.
# Tip: you can test your closing pattern at http://rubular.com
issue_closing_pattern: <%= single_quote(@gitlab_issue_closing_pattern) %>
#}
## Default project features settings
default_projects_features:
issues: {{ cfg('default_projects_features.issues') }}
merge_requests: {{ cfg('default_projects_features.merge_requests') }}
wiki: {{ cfg('default_projects_features.wiki') }}
snippets: {{ cfg('default_projects_features.snippets') }}
builds: {{ cfg('default_projects_features.builds') }}
{# container_registry: <%= @gitlab_default_projects_features_container_registry %> #}
## Webhook settings
# Number of seconds to wait for HTTP response after sending webhook HTTP POST request (default: 10)
webhook_timeout: {{ cfg('webhook_timeout') }}
{# default is just ok
## Repository downloads directory
# When a user clicks e.g. 'Download zip' on a project, a temporary zip file is created in the following directory.
# The default is 'shared/cache/archive/' relative to the root of the Rails app.
# repository_downloads_path: shared/cache/archive/
repository_downloads_path: <%= @gitlab_repository_downloads_path %>
#}
{# we do not support reply by email
## Reply by email
# Allow users to comment on issues and merge requests by replying to notification emails.
# For documentation on how to set this up, see http://doc.gitlab.com/ce/administration/reply_by_email.html
incoming_email:
enabled: <%= @incoming_email_enabled %>
# The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
# The placeholder can be omitted but if present, it must appear in the "user" part of the address (before the `@`).
address: <%= single_quote(@incoming_email_address) %>
# Email account username
# With third party providers, this is usually the full email address.
# With self-hosted email servers, this is usually the user part of the email address.
user: <%= single_quote(@incoming_email_email) %>
# Email account password
password: <%= single_quote(@incoming_email_password) %>
# IMAP server host
host: <%= single_quote(@incoming_email_host) %>
# IMAP server port
port: <%= @incoming_email_port %>
# Whether the IMAP server uses SSL
ssl: <%= @incoming_email_ssl %>
# Whether the IMAP server uses StartTLS
start_tls: <%= @incoming_email_start_tls %>
# The mailbox where incoming mail will end up. Usually "inbox".
mailbox: <%= single_quote(@incoming_email_mailbox_name) %>
# The IDLE command timeout.
idle_timeout: 60
#}
{# we do not support build artifacts
## Build Artifacts
artifacts:
enabled: <%= @artifacts_enabled %>
# The location where Build Artifacts are stored (default: shared/artifacts).
path: <%= @artifacts_path %>
#}
{# we do not support LFS
## Git LFS
lfs:
enabled: <%= @lfs_enabled %>
# The location where LFS objects are stored (default: shared/lfs-objects).
storage_path: <%= @lfs_storage_path %>
#}
{# we do not support container registry
## Container Registry
registry:
enabled: <%= @registry_enabled %>
host: <%= @registry_host %>
port: <%= @registry_port %>
api_url: <%= @registry_api_url %> # internal address to the registry, will be used by GitLab to directly communicate with API
path: <%= @registry_path %>
key: <%= @registry_key_path %>
issuer: <%= @registry_issuer %>
#}
{# we do not support Pages
## GitLab Pages (EE only)
pages:
enabled: <%= @pages_enabled %>
path: <%= @pages_path %>
host: <%= @pages_host %>
port: <%= @pages_port %>
https: <%= @pages_https %>
external_http: <%= @pages_external_http %>
external_https: <%= @pages_external_https %>
#}
{# we do not support Elasticsearch
## Elasticsearch (EE only)
# Enable it if you are going to use elasticsearch instead of
# regular database search
elasticsearch:
enabled: <%= @elasticsearch_enabled %>
host: <%= @elasticsearch_host %>
port: <%= @elasticsearch_port %>
#}
## Gravatar
## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html
gravatar:
{# default is just ok
# gravatar urls: possible placeholders: %{hash} %{size} %{email}
plain_url: <%= single_quote(@gravatar_plain_url) %> # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon
ssl_url: <%= single_quote(@gravatar_ssl_url) %> # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon
#}
## Sidekiq
sidekiq:
log_format: json # (default is the original format)
{# XXX cron jobs are disabled for now - we do not support CI and EE features or we are ok with defaults
## Auxiliary jobs
# Periodically executed jobs, to self-heal GitLab, do external synchronizations, etc.
# Please read here for more information: https://github.com/ondrejbartas/sidekiq-cron#adding-cron-job
cron_jobs:
# Flag stuck CI builds as failed
stuck_ci_builds_worker:
cron: <%= @stuck_ci_builds_worker_cron %>
# Remove outdated repository archives
repository_archive_cache_worker:
cron: <%= @repository_archive_cache_worker_cron %>
##
# GitLab EE only jobs:
# Snapshot active users statistics
historical_data_worker:
cron: <%= @historical_data_worker_cron %>
# Update mirrored repositories
update_all_mirrors_worker:
cron: <%= @update_all_mirrors_worker_cron %>
# Update remote mirrors
update_all_remote_mirrors_worker:
cron: <%= @update_all_remote_mirrors_worker_cron %>
# In addition to refreshing users when they log in,
# periodically refresh LDAP users membership.
# NOTE: This will only take effect if LDAP is enabled
ldap_sync_worker:
cron: <%= @ldap_sync_worker_cron %>
# Gitlab Geo nodes notification worker
# NOTE: This will only take effect if Geo is enabled
geo_bulk_notify_worker:
cron: <%= @geo_bulk_notify_worker_cron %>
#}
#
# 2. GitLab CI settings
# ==========================
{# we do not support CI
gitlab_ci:
# Default project notifications settings:
#
# Send emails only on broken builds (default: true)
all_broken_builds: <%= @gitlab_ci_all_broken_builds %>
#
# Add pusher to recipients list (default: false)
add_pusher: <%= @gitlab_ci_add_pusher || @gitlab_ci_add_committer %>
# The location where build traces are stored (default: builds/). Relative paths are relative to Rails.root
builds_path: <%= @builds_directory %>
#}
#
# 3. Auth settings
# ==========================
## LDAP settings
# You can inspect a sample of the LDAP users with login access by running:
# bundle exec rake gitlab:ldap:check RAILS_ENV=production
ldap:
enabled: false
{# just disabled
enabled: <%= @ldap_enabled %>
sync_time: <%= @ldap_sync_time %>
<% if @ldap_servers.any? %>
servers:
<% @ldap_servers.each do |provider_id, settings| %>
<%= provider_id %>: <%= settings.to_json %>
<% end %>
<% else %>
host: <%= single_quote(@ldap_host) %>
port: <%= @ldap_port %>
uid: <%= single_quote(@ldap_uid) %>
method: <%= single_quote(@ldap_method) %> # "tls" or "ssl" or "plain"
bind_dn: <%= single_quote(@ldap_bind_dn) %>
password: <%= single_quote(@ldap_password) %>
active_directory: <%= @ldap_active_directory %>
allow_username_or_email_login: <%= @ldap_allow_username_or_email_login %>
base: <%= single_quote(@ldap_base) %>
user_filter: <%= single_quote(@ldap_user_filter) %>
## EE only
group_base: <%= single_quote(@ldap_group_base) %>
admin_group: <%= single_quote(@ldap_admin_group) %>
sync_ssh_keys: <%= single_quote(@ldap_sync_ssh_keys) %>
sync_time: <%= @ldap_sync_time %>
<% end %>
#}
## Kerberos settings
kerberos:
enabled: false
{# just disabled
# Allow the HTTP Negotiate authentication method for Git clients
enabled: <%= @kerberos_enabled %>
# Kerberos 5 keytab file. The keytab file must be readable by the GitLab user,
# and should be different from other keytabs in the system.
# (default: use default keytab from Krb5 config)
keytab: <%= @kerberos_keytab %>
# The Kerberos service name to be used by GitLab.
# (default: accept any service name in keytab file)
service_principal_name: <%= @kerberos_service_principal_name %>
# Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails.
# To support both Basic and Negotiate methods with older versions of Git, configure
# nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines
# to dedicate this port to Kerberos authentication. (default: false)
use_dedicated_port: <%= @kerberos_use_dedicated_port %>
port: <%= @kerberos_port %>
https: <%= @kerberos_https %>
#}
## OmniAuth settings
omniauth:
enabled: false
{# just disabled
# Allow login via Twitter, Google, etc. using OmniAuth providers
enabled: <%= @omniauth_enabled %>
# Uncomment this to automatically sign in with a specific omniauth provider's without
# showing GitLab's sign-in page (default: show the GitLab sign-in page)
auto_sign_in_with_provider: <%= @omniauth_auto_sign_in_with_provider %>
# CAUTION!
# This allows users to login without having a user account first. Define the allowed
# providers using an array, e.g. ["saml", "twitter"]
# User accounts will be created automatically when authentication was successful.
allow_single_sign_on: <%= @omniauth_allow_single_sign_on.to_json %>
# Locks down those users until they have been cleared by the admin (default: true).
block_auto_created_users: <%= @omniauth_block_auto_created_users %>
# Look up new users in LDAP servers. If a match is found (same uid), automatically
# link the omniauth identity with the LDAP account. (default: false)
auto_link_ldap_user: <%= @omniauth_auto_link_ldap_user %>
# Allow users with existing accounts to login and auto link their account via SAML
# login, without having to do a manual login first and manually add SAML
# (default: false)
auto_link_saml_user: <%= @omniauth_auto_link_saml_user.to_json %>
# Set different Omniauth providers as external so that all users creating accounts
# via these providers will not be able to have access to internal projects. You
# will need to use the full name of the provider, like `google_oauth2` for Google.
# Refer to the examples below for the full names of the supported providers.
# (default: [])
external_providers: <%= @omniauth_external_providers.to_json %>
## Auth providers
# Uncomment the following lines and fill in the data of the auth provider you want to use
# If your favorite auth provider is not listed you can use others:
# see https://github.com/gitlabhq/gitlab-public-wiki/wiki/Custom-omniauth-provider-configurations
# The 'app_id' and 'app_secret' parameters are always passed as the first two
# arguments, followed by optional 'args' which can be either a hash or an array.
# Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html
providers:
# - { name: 'google_oauth2', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# args: { access_type: 'offline', approval_prompt: '' } }
# - { name: 'twitter', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET'}
# - { name: 'github', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# args: { scope: 'user:email' } }
<% @omniauth_providers.each do |provider| %>
- <%= provider.to_json %>
<% end %>
#}
{# default ($RAILS_ROOT/shared/) is ok - we symlinked it to proper place
# Shared file storage settings
shared:
path: <%= @shared_path %>
#}
# Gitaly settings
gitaly:
# Default Gitaly authentication token. Can be overriden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly.
token:
#
# 4. Advanced settings
# ==========================
## Repositories settings
repositories:
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# IMPORTANT: None of the path components may be symlink, because
# gitlab-shell invokes Dir.pwd inside the repository path and that results
# real path not the symlink.
storages: # You must have at least a `default` storage path.
default:
path: {{ gitlab.repositories }}
gitaly_address: unix:{{ gitaly.socket }} # TCP connections are supported too (e.g. tcp://host:port). TLS connections are also supported using the system certificate pool (eg: tls://host:port).
# gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
## Backup settings
backup:
path: "{{ gitlab.backup }}" # Relative paths are relative to Rails.root (default: tmp/backups/)
{# default permission is ok
archive_permissions: <%= @backup_archive_permissions %> # Permissions for the resulting backup.tar file (default: 0600)
#}
keep_time: {{ cfg('backup_keep_time') }} # default: 0 (forever) (in seconds)
{# default to backup all schemas is just ok
pg_schema: <%= @backup_pg_schema %> # default: nil, it means that all schemas will be backed up
#}
upload:
{# we don't want to upload backup anywhere by gitlab builtin mechanisms
# Fog storage connection settings, see http://fog.io/storage/ .
connection: <%= @backup_upload_connection.to_json if @backup_upload_connection %>
# The remote 'directory' to store your backups. For S3, this would be the bucket name.
remote_directory: <%= single_quote(@backup_upload_remote_directory) %>
multipart_chunk_size: <%= @backup_multipart_chunk_size %>
encryption: <%= @backup_encryption %>
#}
## GitLab Shell settings
gitlab_shell:
path: {{ gitlab_shell_work.location }}
authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
repos_path: {{ gitlab.repositories }}
hooks_path: {{ gitlab_shell_work.location }}/hooks/
secret_file: {{ gitlab_shell.secret }}
# Git over HTTP
upload_pack: true
receive_pack: true
# Git import/fetch timeout, in seconds. Defaults to 3 hours.
# git_timeout: 10800
{# Git over SSH is disabled elsewhere (so we don't care about ssh_port)
# If you use non-standard ssh port you need to specify it
ssh_port: <%= @gitlab_shell_ssh_port %>
#}
# git-annex support (EE only)
# If this setting is set to true, the same setting in config.yml of
# gitlab-shell needs to be set to true
git_annex_enabled: <%= @git_annex_enabled %>
## Git settings
# CAUTION!
# Use the default values unless you really know what you are doing
git:
bin_path: {{ git }}
# The next value is the maximum memory size grit can use
# Given in number of bytes per git object (e.g. a commit)
# This value can be increased if you have very large commits
max_size: {{ cfg('git_max_size') }}
# Git timeout to read a commit, in seconds
timeout: {{ cfg('git_timeout') }}
#
# 5. Extra customization
# ==========================
extra:
{# we do not use google analytics
<% if @extra_google_analytics_id %>
## Google analytics. Uncomment if you want it
google_analytics_id: <%= single_quote(@extra_google_analytics_id) %>
<% end %>
#}
{# we do not use piwik
<% if @extra_piwik_url %>
## Piwik analytics.
piwik_url: <%= single_quote(@extra_piwik_url) %>
piwik_site_id: <%= single_quote(@extra_piwik_site_id) %>
<% end %>
#}
{# we are ok (for now) with default rack-attack git settings
rack_attack:
git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %>
#}
## Site ICP License
# XXX unquote needed only for slapos.core earlier than
# https://lab.nexedi.com/nexedi/slapos.core/commit/347d33d6
# for now we have a lot of old slapos.core deployed...
{% if cfg('icp_license') != '' -%}
ICP: {{ urllib.unquote_plus( str(cfg('icp_license')) ).decode('utf-8') }}
{# ICP: '{{ cfg("icp_license") }}' #}
{% endif %}
development:
<<: *base
test:
<<: *base
gravatar:
enabled: true
gitlab:
host: localhost
port: 80
# When you run tests we clone and setup gitlab-shell
# In order to setup it correctly you need to specify
# your system username you use to run GitLab
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
gitlab_shell:
path: tmp/tests/gitlab-shell/
repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
title: "Redmine"
project_url: "http://redmine/projects/:issues_tracker_id"
issues_url: "http://redmine/:project_id/:issues_tracker_id/:id"
new_issue_url: "http://redmine/projects/:issues_tracker_id/issues/new"
ldap:
enabled: false
servers:
main:
label: ldap
host: 127.0.0.1
port: 3890
uid: 'uid'
method: 'plain' # "tls" or "ssl" or "plain"
base: 'dc=example,dc=com'
user_filter: ''
group_base: 'ou=groups,dc=example,dc=com'
admin_group: ''
sync_ssh_keys: false
staging:
<<: *base
nginx-gitlab-http.conf.in 0000664 0000000 0000000 00000020335 13712426147 0033030 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn with context %}
## GitLab
## Modified from https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl & https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab
##
## Lines starting with two hashes (##) are comments with information.
## Lines starting with one hash (#) are configuration parameters that can be uncommented.
##
##################################
## CHUNKED TRANSFER ##
##################################
##
## It is a known issue that Git-over-HTTP requires chunked transfer encoding [0]
## which is not supported by Nginx < 1.3.9 [1]. As a result, pushing a large object
## with Git (i.e. a single large file) can lead to a 411 error. In theory you can get
## around this by tweaking this configuration file and either:
## - installing an old version of Nginx with the chunkin module [2] compiled in, or
## - using a newer version of Nginx.
##
## At the time of writing we do not know if either of these theoretical solutions works.
## As a workaround users can use Git over SSH to push large files.
##
## [0] https://git.kernel.org/cgit/git/git.git/tree/Documentation/technical/http-protocol.txt#n99
## [1] https://github.com/agentzh/chunkin-nginx-module#status
## [2] https://github.com/agentzh/chunkin-nginx-module
##
###################################
## configuration ##
###################################
upstream gitlab-workhorse {
server unix:{{ gitlab_workhorse.socket }};
}
{# not needed for us - the frontend can do the redirection and also
gitlab/nginx speaks HSTS on https port so when we access https port via http
protocol, it gets redirected to https
<% if @https && @redirect_http_to_https %>
## Redirects all HTTP traffic to the HTTPS host
server {
<% @listen_addresses.each do |listen_address| %>
listen <%= listen_address %>:<%= @redirect_http_to_https_port %>;
<% end %>
server_name <%= @fqdn %>;
server_tokens off; ## Don't show the nginx version number, a security best practice
return 301 https://<%= @fqdn %>:<%= @port %>$request_uri;
access_log <%= @log_directory %>/gitlab_access.log gitlab_access;
error_log <%= @log_directory %>/gitlab_error.log;
}
<% end %>
#}
server {
listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl http2{% endif %};
{# we don't use: kerbeeros
<% if @kerberos_enabled && @kerberos_use_dedicated_port %>
listen <%= listen_address %>:<%= @kerberos_port %><% if @kerberos_https %> ssl<% end %>;
<% end %>
#}
server_name {{ fqdn }};
server_tokens off; ## Don't show the nginx version number, a security best practice
## Increase this if you want to upload large attachments
## Or if you want to accept large git objects over http
client_max_body_size {{ cfg('nginx_client_max_body_size') }};
{% if cfg_https %}
## Strong SSL Security
## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
ssl on;
ssl_certificate {{ nginx.cert_file }};
ssl_certificate_key {{ nginx.key_file }};
{# we don't need - most root CA will be included by default
<% if @ssl_client_certificate %>
ssl_client_certificate <%= @ssl_client_certificate%>;
<% end %>
#}
# GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
# NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
# directly connects to frontend
ssl_ciphers '{{ cfg("nginx_ssl_ciphers") }}';
ssl_protocols {{ cfg('nginx_ssl_protocols') }};
ssl_prefer_server_ciphers {{ cfg('nginx_ssl_prefer_server_ciphers') }};
ssl_session_cache {{ cfg('nginx_ssl_session_cache') }};
ssl_session_timeout {{ cfg('nginx_ssl_session_timeout') }};
{# we do not use: ssl_dhparam
<% if @ssl_dhparam %>
ssl_dhparam <%= @ssl_dhparam %>;
<% end %>
#}
{% endif %}
## Real IP Module Config
## http://nginx.org/en/docs/http/ngx_http_realip_module.html
{% if '{{ cfg("nginx_real_ip_header") }}' %}
real_ip_header '{{ cfg("nginx_real_ip_header") }}';
{% endif %}
{% if '{{ cfg("nginx_real_ip_recursive") }}' %}
real_ip_recursive '{{ cfg("nginx_real_ip_recursive") }}';
{% endif %}
{% for trusted_address in cfg("nginx_real_ip_trusted_addresses").split() %}
set_real_ip_from {{ trusted_address }};
{% endfor %}
## HSTS Config
## https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/
{% if cfg("nginx_hsts_max_age") > 0 -%}
{% if '{{ cfg("nginx_hsts_include_subdomains") }}' == 'true' -%}
add_header Strict-Transport-Security "max-age={{ cfg('nginx_hsts_max_age') }}; includeSubDomains"
{% else -%}
add_header Strict-Transport-Security "max-age={{ cfg('nginx_hsts_max_age') }}";
{% endif -%}
{% endif -%}
## Individual nginx logs for this GitLab vhost
access_log {{ nginx.log }}/gitlab_access.log gitlab_access;
error_log {{ nginx.log }}/gitlab_error.log;
# Set CORS header
add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
add_header 'Access-Control-Allow-Credentials' true;
#{{ 'gzip off;' if cfg_https else ''}}
{% if '{{ cfg("nginx_gzip_enabled") }}' == 'true' -%}
gzip on;
gzip_static on;
gzip_comp_level 2;
gzip_http_version 1.1;
gzip_vary on;
gzip_disable "msie6";
gzip_min_length 10240;
gzip_proxied no-cache no-store private expired auth;
gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/json application/xml application/rss+xml;
{% endif -%}
## https://github.com/gitlabhq/gitlabhq/issues/694
## Some requests take more than 30 seconds.
proxy_read_timeout {{ cfg('nginx_proxy_read_timeout') }};
proxy_connect_timeout {{ cfg('nginx_proxy_connect_timeout') }};
proxy_redirect off;
proxy_http_version 1.1;
{# we do not support relative URL - path is always "/" #}
{% set path = "/" %}
#if ($http_host = "") {
# set $http_host_with_default "<%= default_host %>";
#}
#if ($http_host != "") {
# set $http_host_with_default $http_host;
#}
location ~ (\.git/gitlab-lfs/objects|\.git/info/lfs/objects/batch$) {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
{% if cfg_https %}
proxy_set_header X-Forwarded-Ssl on;
{% endif %}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }};
proxy_pass http://gitlab-workhorse;
}
location {{ path }} {
# NOTE(slapos) proxy headers are defined upstream in omnibus-gitlab in:
# - files/gitlab-config-template/gitlab.rb.template nginx['proxy_set_headers']
# - files/gitlab-cookbooks/gitlab/attributes/default.rb default['gitlab']['nginx']['proxy_set_headers']
# - files/gitlab-cookbooks/gitlab/libraries/gitlab.rb parse_nginx_proxy_headers()
# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
if ($request_method = OPTIONS ) {
add_header Allow "GET, OPTIONS";
add_header Content-Type text/plain;
add_header 'Access-Control-Allow-Origin' $http_origin;
add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Authorization, Content-Type, Accept";
return 200;
}
proxy_cache off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
{% if cfg_https %}
proxy_set_header X-Forwarded-Ssl on;
{% endif %}
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto {{ "https" if cfg_https else "http" }};
proxy_pass http://gitlab-workhorse;
}
location ~ ^/(assets)/ {
proxy_cache off;
proxy_pass http://gitlab-workhorse;
}
error_page 404 /404.html;
error_page 422 /422.html;
error_page 500 /500.html;
error_page 502 /502.html;
location ~ ^/(404|422|500|502)\.html$ {
root {{ gitlab_work.location }}/public;
internal;
}
{# we don't support custom nginx configs
<%= @custom_gitlab_server_config %>
#}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/nginx.conf.in 0000664 0000000 0000000 00000006560 13712426147 0030676 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %}
# user directive makes sense only when running initially as root
# (and nginx will complain if not and directive given)
# user {{ backend_info.user }};
worker_processes {{ cfg('nginx_worker_processes') }};
error_log stderr;
pid {{ directory.run }}/nginx.pid;
daemon off;
events {
worker_connections {{ cfg('nginx_worker_connections') }};
}
http {
log_format gitlab_access '{{ cfg("nginx_log_format") }}';
{# we do not use: ci, mattermost
log_format gitlab_ci_access '<%= @gitlab_ci_access_log_format %>';
log_format gitlab_mattermost_access '<%= @gitlab_mattermost_access_log_format %>';
#}
sendfile {{ cfg('nginx_sendfile') }};
tcp_nopush {{ cfg('nginx_tcp_nopush') }};
tcp_nodelay {{ cfg('nginx_tcp_nodelay') }};
keepalive_timeout {{ cfg('nginx_keepalive_timeout') }};
gzip {{ cfg('nginx_gzip') }};
gzip_http_version {{ cfg('nginx_gzip_http_version') }};
gzip_comp_level {{ cfg('nginx_gzip_comp_level') }};
gzip_proxied {{ cfg('nginx_gzip_proxied') }};
gzip_types {{ cfg('nginx_gzip_types') }};
include {{ nginx_mime_types }};
{# we do not do nginx caching:
- gitlab-workhorse serves raw blobs fast
- we have caches on frontend nodes
proxy_cache_path <%= @proxy_cache_path %>;
proxy_cache <%= @proxy_cache %>;
#}
include {{ nginx_gitlab_http_conf }};
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
# Remove private_token from the request URI
# In: /foo?private_token=unfiltered&authenticity_token=unfiltered&rss_token=unfiltered&...
# Out: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
map $request_uri $temp_request_uri_1 {
default $request_uri;
~(?i)^(?.*)(?[\?&]private[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest";
}
# Remove authenticity_token from the request URI
# In: /foo?private_token=[FILTERED]&authenticity_token=unfiltered&rss_token=unfiltered&...
# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
map $temp_request_uri_1 $temp_request_uri_2 {
default $temp_request_uri_1;
~(?i)^(?.*)(?[\?&]authenticity[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest";
}
# Remove rss_token from the request URI
# In: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=unfiltered&...
# Out: /foo?private_token=[FILTERED]&authenticity_token=[FILTERED]&rss_token=[FILTERED]&...
map $temp_request_uri_2 $filtered_request_uri {
default $temp_request_uri_2;
~(?i)^(?.*)(?[\?&]rss[\-_]token)=[^&]*(?.*)$ "$start$temp=[FILTERED]$rest";
}
# A version of the referer without the query string
map $http_referer $filtered_http_referer {
default $http_referer;
~^(?.*)\? $temp;
}
{# we don't need: ci, pages, mattermost, registry
include <%= @gitlab_ci_http_config %>
include <%= @gitlab_pages_http_config %>;
include <%= @gitlab_mattermost_http_config %>
include <%= @gitlab_registry_http_config %>;
#}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/rack_attack.rb.in 0000664 0000000 0000000 00000003026 13712426147 0031472 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %}
# 1. Rename this file to rack_attack.rb
# 2. Review the paths_to_be_protected and add any other path you need protecting
#
paths_to_be_protected = [
"#{Rails.application.config.relative_url_root}/users/password",
"#{Rails.application.config.relative_url_root}/users/sign_in",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
"#{Rails.application.config.relative_url_root}/users",
"#{Rails.application.config.relative_url_root}/users/confirmation",
"#{Rails.application.config.relative_url_root}/unsubscribes/"
]
# Create one big regular expression that matches strings starting with any of
# the paths_to_be_protected.
paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
unless Rails.env.test? || !rack_attack_enabled
Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
if req.post? && req.path =~ paths_regex
req.ip
end
end
end
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/resque.yml.in 0000664 0000000 0000000 00000000513 13712426147 0030723 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
# (last udpdated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
production: unix://{{ redis.unixsocket }}
smtp_settings.rb.in 0000664 0000000 0000000 00000002411 13712426147 0032044 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_bool with context %}
{% if cfg_bool('smtp_enable') %}
if Rails.env.production?
Gitlab::Application.config.action_mailer.delivery_method = :smtp
ActionMailer::Base.delivery_method = :smtp
ActionMailer::Base.smtp_settings = {
address: "{{ cfg('smtp_address') }}",
port: {{ cfg('smtp_port') }},
user_name: "{{ cfg('smtp_user_name') }}",
password: "{{ cfg('smtp_password') }}",
domain: "{{ cfg('smtp_domain') }}",
authentication: :{{ cfg('smtp_authentication') }},
enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }},
# ssl:
openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}'
# ca_path:
# ca_file:
}
end
{% else %}
# SMTP disabled in instance configuration (see `smtp_enable` parameter).
# Mail sending, if enabled (see `email_enabled`), will be done via sendmail.
{% endif %}
template-gitlab-resiliency-restore.sh.in 0000664 0000000 0000000 00000006460 13712426147 0036060 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template #!{{ bash_bin }}
# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
# DaTA WILL BE ERASED
set -e
echo "###################################################################################"
echo "# #"
echo "# Warning: DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE DaTA WILL BE ERASED !!! #"
echo "# #"
echo "###################################################################################"
echo -e "\nWill start in 10 seconds, cancel execution if you didn't want to run this script."
sleep 10
postgres_executable="{{ postgress_script }}"
redis_executable="{{ redis_script }}"
git_backup_directory="{{ gitlab_backup_dir }}"
redis_pid_file="{{ redis_pid_file }}"
postgres_pid_file="{{ postgres_pid_file }}"
bin_location="{{ bin_directory }}"
run_location="{{ run_directory }}"
git_location="{{ git_location }}"
go_work_bin="{{ go_work_bin }}"
etc_location="{{ etc_directory }}"
gitlab_work="{{ gitlab_work_location }}"
promise_check="{{ promise_lab_location }}"
unicorn_script="{{ unicorn_script }}"
sidekiq_script="{{ sidekiq_script }}"
# export GIT_EXEC_PATH=$git_location/libexec/git-core/
check_process () {
pid_file=$1
pname=$2
if [ -e "$pid_file" ]; then
pid=$(head -n 1 $pid_file) > /dev/null 2>&1
if kill -0 "$pid"; then
echo "$pname is already running with pid $pid. Aborting."
exit 1
fi
fi
}
kill_process () {
pid=$1
R=0
kill -0 "$pid" > /dev/null 2>&1 || R=$?
if [ $R -eq 0 ]; then
kill -TERM $pid
fi
}
check_process $postgres_pid_file "Postgres"
check_process $redis_pid_file "Redis"
check_process $run_location/unicorn.pid "Unicorn"
if [ -f "$postgres_pid_file" ]; then
rm $postgres_pid_file
fi
echo "Starting Postgres..."
$postgres_executable &
postgres_pid=$!
trap "echo 'kill $postgres_pid" EXIT TERM INT
echo "Starting Redis server..."
$redis_executable &
redis_pid=$!
trap "kill $postgres_pid $redis_pid" EXIT TERM INT
echo "[OK]"
echo "Restoring gitlab data..."
# XXX - workaround until this problem is fixed on runner1
sed -ie "s/connection.execute('TRUNCATE schema_migrations')\s*$/connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'/g" $gitlab_work/lib/tasks/gitlab/db.rake
cd $git_backup_directory
PATH=$bin_location:$go_work_bin:$git_location/bin:$PATH gitlab-backup restore -vupok -go HEAD
echo "Checking gitlab promises..."
echo "[info] Not all promises are checked!"
$promise_check/gitlab-app
echo "Starting Unicorn to check gitlab-shell promise..."
$unicorn_script &
unicorn_pid=$!
trap "kill $postgres_pid $redis_pid $unicorn_pid" EXIT TERM INT
sleep 60
if [ -s "$run_location/unicorn.pid" ]; then
unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $unicorn_ppid" EXIT TERM INT
fi
$promise_check/gitlab-shell
#echo "starting Sidekiq to check sidekiq promise..."
#$sidekiq_script &
#sidekiq_pid=$!
#trap "kill $sidekiq_pid" EXIT TERM INT
#$promise_check/sidekiq
kill_process $postgres_pid
kill_process $redis_pid
kill_process $unicorn_pid
RESTORE_EXIT_CODE=$?
if [ $RESTORE_EXIT_CODE -eq 0 ]; then
echo 'Backup restoration successfully completed.'
else
echo 'Backup restoration failed.'
fi
exit $RESTORE_EXIT_CODE
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/template/unicorn.rb.in 0000664 0000000 0000000 00000011201 13712426147 0030672 0 ustar 00root root 0000000 0000000 {{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg with context %}
# What ports/sockets to listen on, and what options for them.
# we listen only on unix socket
listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
#listen "127.0.0.1:8888", :tcp_nopush => true
working_directory '{{ gitlab_work.location }}'
# What the timeout for killing busy workers is, in seconds
timeout {{ cfg('unicorn_worker_timeout') }}
# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
preload_app true
GC.respond_to?(:copy_on_write_friendly=) and
GC.copy_on_write_friendly = true
# Enable this flag to have unicorn test client connections by writing the
# beginning of the HTTP headers before calling the application. This
# prevents calling the application for connections that have disconnected
# while queued. This is only guaranteed to detect clients on the same
# host unicorn runs on, and unlikely to detect disconnects even on a
# fast LAN.
check_client_connection false
# How many worker processes
worker_processes {{ cfg('unicorn_worker_processes') }}
# about before_fork / after_fork - see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
# http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
# What to do before we fork a worker
before_fork do |server, worker|
# XXX why gitlab does not enable this?
# # the following is highly recomended for Rails + "preload_app true"
# # as there's no need for the master process to hold a connection
# defined?(ActiveRecord::Base) and
# ActiveRecord::Base.connection.disconnect!
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
# twice as many worker_processes as you have configured.
#
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
#
# Throttle the master from forking too quickly by sleeping. Due
# to the implementation of standard Unix signal handlers, this
# helps (but does not completely) prevent identical, repeated signals
# from being lost when the receiving process is busy.
# sleep 1
end
# What to do after we fork a worker
after_fork do |server, worker|
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
# XXX why gitlab does not enable this?
# # the following is *required* for Rails + "preload_app true",
# defined?(ActiveRecord::Base) and
# ActiveRecord::Base.establish_connection
# reset prometheus client, this will cause any opened metrics files to be closed
#defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
# Prometheus::Client.reinitialize_on_pid_change
# if preload_app is true, then you may also want to check and
# restart any other shared sockets/descriptors such as Memcached,
# and Redis. TokyoCabinet file handles are safe to reuse
# between any number of forked children (assuming your kernel
# correctly implements pread()/pwrite() system calls)
end
# Where to drop a pidfile
pid '{{ directory.run }}/unicorn.pid'
# Where stderr gets logged
stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
# Where stdout gets logged
stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
{# we do not support Relative url
<%- if @relative_url %>
# Relative url from where GitLab is served
ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
<%- end %>
#}
# Min memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
# Max memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/test/ 0000775 0000000 0000000 00000000000 13712426147 0025434 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/test/README.md 0000664 0000000 0000000 00000000042 13712426147 0026707 0 ustar 00root root 0000000 0000000 Tests for Gitlab software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/test/setup.py 0000664 0000000 0000000 00000003657 13712426147 0027161 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.gitlab'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Gitlab",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'requests',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/test/test.py 0000664 0000000 0000000 00000004120 13712426147 0026762 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import logging
from six.moves.urllib.parse import urlparse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestGitlab(SlapOSInstanceTestCase):
__partition_reference__ = 'G' # solve path too long for postgresql and unicorn
@classmethod
def getInstanceSoftwareType(cls):
return 'gitlab-test'
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend_url']
def test_http_get(self):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/gitlab/watcher.in 0000664 0000000 0000000 00000003466 13712426147 0026453 0 ustar 00root root 0000000 0000000 #!{{ bash.location }}/bin/bash
# run program under watchdog
# watcher [ ...]
#
# = code1,code2,...
#
# if the program terminates with status in - it is restarted after grace period.
# if the program terminates otherwise - whole process terminates.
#
# code can be numeric or symbolic - refering to a signal name. example:
#
# watcher 0,SIGKILL ...
die() {
echo "$@" 1>&2
exit 1
}
if [ "$#" -lt 2 ]; then
die "Usage: watcher [ ...]"
fi
restart_codes="$1"; shift
prog="$@"
# signumber -> #sig
signumber() {
signame=$1
# "11) SIGSEGV "
sigentry=`kill -l |grep -o "[0-9]\+) $signame\(\s\|$\)"` ||
die "E: $signame is not a signal"
echo "$sigentry" | grep -o "[0-9]\+"
}
# restart codes as set
declare -A restarts
for code in `echo "$restart_codes" |sed 's/,/ /g'`; do
case $code in
*[!0-9]*)
# non-number - treat it as signal name
signo=`signumber $code` || exit 1
code=$((128 + $signo)) # exit code of process terminated by signal #signo
;;
*)
# already number
;;
esac
restarts[$code]=y
done
progpid=""
# make sure to terminate children, when we exit.
# needed for e.g. when `slapos node stop ...` kills us.
trap 'atexit' EXIT
atexit() {
jobs="$(jobs -p)"
test -n "$jobs" && kill $jobs
}
# run prog under monitoring
while true; do
echo "run $prog"
$prog &
progpid=$!
echo "wait $progpid"
wait $progpid
status=$?
echo "-> $status"
# if program terminated not with expected status - exit
if [ "${restarts[$status]}" != y ] ; then
echo "exit $status"
exit "$status"
fi
# otherwise sleep a bit and restart
sleep 1
done
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/ 0000775 0000000 0000000 00000000000 13712426147 0024612 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/README.md 0000664 0000000 0000000 00000003172 13712426147 0026074 0 ustar 00root root 0000000 0000000 # grafana / telegraf / influxdb
This is an experimental integration, mainly to evaluate these solutions.
## Custom telegraf plugins
See https://github.com/influxdata/telegraf to learn about plugins.
Useful plugins in this context are probably
[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser)
or
[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http).
Telegraf will save in the `telegraf` database from the embedded influxdb server.
## Grafana
A default user is created, username and password are published as connection
parameters. You can add more users in grafana interface.
Datasources should be automatically added.
## Influxdb
Influxdb backups are not done automatically by this software release.
One important thing to notice is that the backup protocol is enabled on ipv4
provided by slapos, so make sure this ip is not reachable from untrusted
sources.
# Ingesting/Visualizing logs
Eventhough main feature is visualizing metrics, Grafana has a feature called "Explore" to view logs for a time frame.
The following backend can be used:
## Loki
See `TestLoki` in test for an example.
## Influxdb
Influxdb logs only have tags and there does not seem to be a way to search (except than tag and time frame).
To inject log files containing:
```
INFO the message
WARN another message
```
use config like:
```
[[inputs.logparser]]
files = ["/tmp/x*.log", "/tmp/aaa.log"]
[inputs.logparser.grok]
measurement = "logs"
patterns = ['^%{WORD:level:tag} %{GREEDYDATA:message:string}']
```
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/buildout.hash.cfg 0000664 0000000 0000000 00000002640 13712426147 0030046 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = 13f4f3806522b265dd1912ff169d146d
[influxdb-config-file]
filename = influxdb-config-file.cfg.in
md5sum = a28972ced3e0f4aa776e43a9c44717c0
[telegraf-config-file]
filename = telegraf-config-file.cfg.in
md5sum = a1a9c22c2a7829c66a49fc2504604d21
[grafana-config-file]
filename = grafana-config-file.cfg.in
md5sum = 8244d430905b968795c7946049bed9e3
[grafana-provisioning-config-file]
filename = grafana-provisioning-config-file.cfg.in
md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
[loki-config-file]
filename = loki-config-file.cfg.in
md5sum = 02ba5acf23fcf88f5594919f46838533
[promtail-config-file]
filename = promtail-config-file.cfg.in
md5sum = c77788d0a3cc654ad9393eb4b1f31e94
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/gowork.cfg 0000664 0000000 0000000 00000002345 13712426147 0026607 0 ustar 00root root 0000000 0000000 # Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_golang_dep:recipe}
${go_github.com_grafana_grafana:recipe}
${go_github.com_grafana_loki:recipe}
${go_github.com_influxdata_influxdb:recipe}
${go_github.com_influxdata_telegraf:recipe}
[go_github.com_golang_dep]
<= go-git-package
go.importpath = github.com/golang/dep
repository = https://github.com/golang/dep
revision = 1f7c19e5f5
[go_github.com_grafana_grafana]
<= go-git-package
go.importpath = github.com/grafana/grafana
repository = https://github.com/grafana/grafana
revision = v6.3.0-0-g830da0fda0
[go_github.com_grafana_loki]
<= go-git-package
go.importpath = github.com/grafana/loki
repository = https://github.com/grafana/loki
revision = v0.3.0-0-gc673380bb3
[go_github.com_influxdata_influxdb]
<= go-git-package
go.importpath = github.com/influxdata/influxdb
repository = https://github.com/influxdata/influxdb
revision = v1.7.8rc1-0-ga7afa2ecf1
[go_github.com_influxdata_telegraf]
<= go-git-package
go.importpath = github.com/influxdata/telegraf
repository = https://github.com/influxdata/telegraf
revision = 1.11.1-0-gf91de60d37
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/grafana-config-file.cfg.in0000664 0000000 0000000 00000035055 13712426147 0031467 0 ustar 00root root 0000000 0000000 ##################### Grafana Configuration Defaults #####################
#
# Do not modify this file in grafana installs
#
# possible values : production, development
app_mode = production
# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
instance_name = ${HOSTNAME}
#################################### Paths ###############################
[paths]
# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
#data = data
data = {{ grafana['data-dir'] }}
# Directory where grafana can store logs
#logs = data/log
logs = {{ grafana['logs-dir'] }}
# Directory where grafana will automatically scan and look for plugins
#plugins = data/plugins
plugins = {{ grafana['plugins-dir'] }}
# folder that contains provisioning config files that grafana will apply on startup and while running.
#provisioning = conf/provisioning
provisioning = {{ grafana['provisioning-config-dir'] }}
#################################### Server ##############################
[server]
# Protocol (http, https, socket)
protocol = https
# The ip address to bind to, empty will bind to all interfaces
#http_addr =
http_addr = [{{ grafana['ipv6'] }}]
# The http port to use
#http_port = 3000
http_port = {{ grafana['port'] }}
# The public facing domain name used to access grafana from a browser
domain = {{ apache_frontend['connection-domain'] }}
# Redirect to correct domain if host header does not match domain
# Prevents DNS rebinding attacks
enforce_domain = false
# The full public facing url
root_url = {{ apache_frontend['connection-secure_access'] }}
# Log web requests
router_logging = false
# the path relative working path
static_root_path = public
# enable gzip
#enable_gzip = false
enable_gzip = true
# https certs & key file
#cert_file =
cert_file = {{ grafana['ssl-cert-file'] }}
#cert_key =
cert_key = {{ grafana['ssl-key-file'] }}
# Unix socket path
#socket = /tmp/grafana.sock
#################################### Database ############################
[database]
# You can configure the database connection by specifying type, host, name, user and password
# as separate properties or as on string using the url property.
# Either "mysql", "postgres" or "sqlite3", it's your choice
type = sqlite3
host = 127.0.0.1:3306
name = grafana
user = root
# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
password =
# Use either URL or the previous fields to configure the database
# Example: mysql://user:secret@host:port/database
url =
# Max idle conn setting default is 2
max_idle_conn = 2
# Max conn setting default is 0 (mean not set)
max_open_conn =
# Set to true to log the sql calls and execution times.
log_queries =
# For "postgres", use either "disable", "require" or "verify-full"
# For "mysql", use either "true", "false", or "skip-verify".
ssl_mode = disable
ca_cert_path =
client_key_path =
client_cert_path =
server_cert_name =
# For "sqlite3" only, path relative to data_path setting
path = grafana.db
#################################### Session #############################
[session]
# Either "memory", "file", "redis", "mysql", "postgres", "memcache", default is "file"
provider = file
# Provider config options
# memory: not have any config yet
# file: session dir path, is relative to grafana data_path
# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`
# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
# mysql: go-sql-driver/mysql dsn config string, examples:
# `user:password@tcp(127.0.0.1:3306)/database_name`
# `user:password@unix(/var/run/mysqld/mysqld.sock)/database_name`
# memcache: 127.0.0.1:11211
provider_config = sessions
# Session cookie name
cookie_name = grafana_sess
# If you use session in https only, default is false
#cookie_secure = false
cookie_secure = true
# Session life time, default is 86400
session_life_time = 86400
gc_interval_time = 86400
#################################### Data proxy ###########################
[dataproxy]
# This enables data proxy logging, default is false
logging = false
#################################### Analytics ###########################
[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
# No ip addresses are being tracked, only simple counters to track
# running instances, dashboard and error counts. It is very helpful to us.
# Change this option to false to disable reporting.
reporting_enabled = true
# Set to false to disable all checks to https://grafana.com
# for new versions (grafana itself and plugins), check is used
# in some UI views to notify that grafana or plugin update exists
# This option does not cause any auto updates, nor send any information
# only a GET request to https://grafana.com to get latest versions
check_for_updates = true
# Google Analytics universal tracking code, only enabled if you specify an id here
google_analytics_ua_id =
# Google Tag Manager ID, only enabled if you specify an id here
google_tag_manager_id =
#################################### Security ############################
[security]
# default admin user, created on startup
#admin_user = "admin"
admin_user = "{{ grafana['admin-user'] }}"
# default admin password, can be changed before first start of grafana, or in profile settings
#admin_password = admin
admin_password = "{{ grafana['admin-password'] }}"
# used for signing
#secret_key = SW2YcwTIb9zpOOhoPsMm
secret_key = "{{ grafana['secret-key'] }}"
# Auto-login remember days
login_remember_days = 7
cookie_username = grafana_user
cookie_remember_name = grafana_remember
# disable gravatar profile images
disable_gravatar = false
# data source proxy whitelist (ip_or_domain:port separated by spaces)
data_source_proxy_whitelist =
#################################### Snapshots ###########################
[snapshots]
# snapshot sharing options
external_enabled = true
external_snapshot_url = https://snapshots-origin.raintank.io
external_snapshot_name = Publish to snapshot.raintank.io
# remove expired snapshot
snapshot_remove_expired = true
# remove snapshots after 90 days
snapshot_TTL_days = 90
#################################### Dashboards ##################
[dashboards]
# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
versions_to_keep = 20
#################################### Users ###############################
[users]
# disable user signup / registration
allow_sign_up = false
# Allow non admin users to create organizations
allow_org_create = false
# Set to true to automatically assign new users to the default organization (id 1)
auto_assign_org = true
# Default role new users will be automatically assigned (if auto_assign_org above is set to true)
auto_assign_org_role = Viewer
# Require email validation before sign up completes
verify_email_enabled = false
# Background text for the user field on the login page
login_hint = email or username
# Default UI theme ("dark" or "light")
default_theme = dark
# External user management
external_manage_link_url =
external_manage_link_name =
external_manage_info =
# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
viewers_can_edit = false
[auth]
# Set to true to disable (hide) the login form, useful if you use OAuth
disable_login_form = false
# Set to true to disable the signout link in the side menu. useful if you use auth.proxy
disable_signout_menu = false
#################################### Anonymous Auth ######################
[auth.anonymous]
# enable anonymous access
enabled = false
# specify organization name that should be used for unauthenticated users
org_name = Main Org.
# specify role for unauthenticated users
org_role = Viewer
#################################### Github Auth #########################
[auth.github]
enabled = false
allow_sign_up = true
client_id = some_id
client_secret = some_secret
scopes = user:email
auth_url = https://github.com/login/oauth/authorize
token_url = https://github.com/login/oauth/access_token
api_url = https://api.github.com/user
team_ids =
allowed_organizations =
#################################### Google Auth #########################
[auth.google]
enabled = false
allow_sign_up = true
client_id = some_client_id
client_secret = some_client_secret
scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
auth_url = https://accounts.google.com/o/oauth2/auth
token_url = https://accounts.google.com/o/oauth2/token
api_url = https://www.googleapis.com/oauth2/v1/userinfo
allowed_domains =
hosted_domain =
#################################### Grafana.com Auth ####################
# legacy key names (so they work in env variables)
[auth.grafananet]
enabled = false
allow_sign_up = true
client_id = some_id
client_secret = some_secret
scopes = user:email
allowed_organizations =
[auth.grafana_com]
enabled = false
allow_sign_up = true
client_id = some_id
client_secret = some_secret
scopes = user:email
allowed_organizations =
#################################### Generic OAuth #######################
[auth.generic_oauth]
name = OAuth
enabled = false
allow_sign_up = true
client_id = some_id
client_secret = some_secret
scopes = user:email
auth_url =
token_url =
api_url =
team_ids =
allowed_organizations =
#################################### Basic Auth ##########################
[auth.basic]
enabled = true
#################################### Auth Proxy ##########################
[auth.proxy]
enabled = false
header_name = X-WEBAUTH-USER
header_property = username
auto_sign_up = true
ldap_sync_ttl = 60
whitelist =
#################################### Auth LDAP ###########################
[auth.ldap]
enabled = false
config_file = /etc/grafana/ldap.toml
allow_sign_up = true
#################################### SMTP / Emailing #####################
[smtp]
#enabled = false
enabled = {{ slapparameter_dict.get('smtp-server') and 'true' or 'false' }}
#host = locahost:25
host = {{ slapparameter_dict.get('smtp-server', '') }}
#user =
user = {{ slapparameter_dict.get('smtp-username', '') }}
# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
#password =
password = {{ slapparameter_dict.get('smtp-password', '') and '"""%s"""' % slapparameter_dict['smtp-password'] or ""}}
cert_file =
key_file =
#skip_verify = false
skip_verify = {{ slapparameter_dict.get('smtp-verify-ssl', True) and 'false' or 'true' }}
#from_address = admin@grafana.localhost
from_address = {{ slapparameter_dict.get('email-from-address', '') }}
#from_name = Grafana
from_name = {{ slapparameter_dict.get('email-from-name', 'Grafana') }}
ehlo_identity =
[emails]
welcome_email_on_sign_up = false
templates_pattern = emails/*.html
#################################### Logging ##########################
[log]
# Either "console", "file", "syslog". Default is console and file
# Use space to separate multiple modes, e.g. "console file"
mode = console file
# Either "debug", "info", "warn", "error", "critical", default is "info"
level = info
# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
filters =
# For "console" mode only
[log.console]
level =
# log line format, valid options are text, console and json
format = console
# For "file" mode only
[log.file]
level =
# log line format, valid options are text, console and json
format = text
# This enables automated log rotate(switch of following options), default is true
log_rotate = true
# Max line number of single file, default is 1000000
max_lines = 1000000
# Max size shift of single file, default is 28 means 1 << 28, 256MB
max_size_shift = 28
# Segment log daily, default is true
daily_rotate = true
# Expired days of log file(delete after max days), default is 7
max_days = 7
[log.syslog]
level =
# log line format, valid options are text, console and json
format = text
# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
network =
address =
# Syslog facility. user, daemon and local0 through local7 are valid.
facility =
# Syslog tag. By default, the process' argv[0] is used.
tag =
#################################### Usage Quotas ########################
[quota]
enabled = false
#### set quotas to -1 to make unlimited. ####
# limit number of users per Org.
org_user = 10
# limit number of dashboards per Org.
org_dashboard = 100
# limit number of data_sources per Org.
org_data_source = 10
# limit number of api_keys per Org.
org_api_key = 10
# limit number of orgs a user can create.
user_org = 10
# Global limit of users.
global_user = -1
# global limit of orgs.
global_org = -1
# global limit of dashboards
global_dashboard = -1
# global limit of api_keys
global_api_key = -1
# global limit on number of logged in users.
global_session = -1
#################################### Alerting ############################
[alerting]
# Disable alerting engine & UI features
enabled = true
# Makes it possible to turn off alert rule execution but alerting UI is visible
execute_alerts = true
#################################### Internal Grafana Metrics ############
# Metrics available at HTTP API Url /metrics
[metrics]
enabled = true
interval_seconds = 10
# Send internal Grafana metrics to graphite
[metrics.graphite]
# Enable by setting the address setting (ex localhost:2003)
address =
prefix = prod.grafana.%(instance_name)s.
[grafana_net]
url = https://grafana.com
[grafana_com]
url = https://grafana.com
#################################### Distributed tracing ############
[tracing.jaeger]
# jaeger destination (ex localhost:6831)
address =
# tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
always_included_tag =
# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
sampler_type = const
# jaeger samplerconfig param
# for "const" sampler, 0 or 1 for always false/true respectively
# for "probabilistic" sampler, a probability between 0 and 1
# for "rateLimiting" sampler, the number of spans per second
# for "remote" sampler, param is the same as for "probabilistic"
# and indicates the initial sampling rate before the actual one
# is received from the mothership
sampler_param = 1
#################################### External Image Storage ##############
[external_image_storage]
# You can choose between (s3, webdav, gcs, azure_blob)
provider =
[external_image_storage.s3]
bucket_url =
bucket =
region =
path =
access_key =
secret_key =
[external_image_storage.webdav]
url =
username =
password =
public_url =
[external_image_storage.gcs]
key_file =
bucket =
path =
[external_image_storage.azure_blob]
account_name =
account_key =
container_name =
grafana-provisioning-config-file.cfg.in 0000664 0000000 0000000 00000000767 13712426147 0034136 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana # https://grafana.com/docs/administration/provisioning/#example-datasource-config-file
apiVersion: 1
datasources:
- name: telegraf
type: influxdb
access: proxy
url: {{ influxdb['url'] }}
user: {{ influxdb['auth-username'] }}
database: telegraf
isDefault: true
jsonData:
tlsSkipVerify: true
secureJsonData:
password: {{ influxdb['auth-password'] }}
version: 1
editable: false
- name: loki
type: loki
access: proxy
url: {{ loki['url'] }}
version: 1
editable: false
influxdb-config-file.cfg.in 0000664 0000000 0000000 00000005725 13712426147 0031625 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana reporting-disabled = false
bind-address = "[{{ influxdb['local-host'] }}]:{{ influxdb['rpc-port'] }}"
[meta]
dir = "{{ influxdb['data-dir'] }}/meta"
retention-autocreate = true
logging-enabled = true
[data]
dir = "{{ influxdb['data-dir'] }}/data"
index-version = "inmem"
wal-dir = "{{ influxdb['data-dir'] }}/wal"
wal-fsync-delay = "0s"
query-log-enabled = true
cache-max-memory-size = 1073741824
cache-snapshot-memory-size = 26214400
cache-snapshot-write-cold-duration = "10m0s"
compact-full-write-cold-duration = "4h0m0s"
max-series-per-database = 1000000
max-values-per-tag = 100000
max-concurrent-compactions = 0
trace-logging-enabled = false
[coordinator]
write-timeout = "10s"
max-concurrent-queries = 0
query-timeout = "0s"
log-queries-after = "0s"
max-select-point = 0
max-select-series = 0
max-select-buckets = 0
[retention]
enabled = true
check-interval = "30m0s"
[shard-precreation]
enabled = true
check-interval = "10m0s"
advance-period = "30m0s"
[monitor]
store-enabled = true
store-database = "_internal"
store-interval = "10s"
[subscriber]
enabled = true
http-timeout = "30s"
insecure-skip-verify = false
ca-certs = ""
write-concurrency = 40
write-buffer-size = 1000
[http]
enabled = true
bind-address = "[{{ influxdb['host'] }}]:{{ influxdb['http-port'] }}"
auth-enabled = true
log-enabled = true
write-tracing = false
pprof-enabled = true
https-enabled = true
https-certificate = "{{ influxdb['ssl-cert-file'] }}"
https-private-key = "{{ influxdb['ssl-key-file'] }}"
max-row-limit = 0
max-connection-limit = 0
shared-secret = ""
realm = "InfluxDB"
unix-socket-enabled = true
bind-socket = "{{ influxdb['unix-socket'] }}"
max-body-size = 25000000
[ifql]
enabled = false
log-enabled = true
bind-address = ":8082"
[[graphite]]
enabled = false
bind-address = ":2003"
database = "graphite"
retention-policy = ""
protocol = "tcp"
batch-size = 5000
batch-pending = 10
batch-timeout = "1s"
consistency-level = "one"
separator = "."
udp-read-buffer = 0
[[collectd]]
enabled = false
bind-address = ":25826"
database = "collectd"
retention-policy = ""
batch-size = 5000
batch-pending = 10
batch-timeout = "10s"
read-buffer = 0
typesdb = "/usr/share/collectd/types.db"
security-level = "none"
auth-file = "/etc/collectd/auth_file"
parse-multivalue-plugin = "split"
[[opentsdb]]
enabled = false
bind-address = ":4242"
database = "opentsdb"
retention-policy = ""
consistency-level = "one"
tls-enabled = false
certificate = "/etc/ssl/influxdb.pem"
batch-size = 1000
batch-pending = 5
batch-timeout = "1s"
log-point-errors = true
[[udp]]
enabled = false
bind-address = ":8089"
database = "udp"
retention-policy = ""
batch-size = 5000
batch-pending = 10
read-buffer = 0
batch-timeout = "1s"
precision = ""
[continuous_queries]
log-enabled = true
enabled = true
query-stats-enabled = false
run-interval = "1s"
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/instance-input-schema.json0000664 0000000 0000000 00000002331 13712426147 0031703 0 ustar 00root root 0000000 0000000 {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Parameters to instantiate Grafana",
"additionalProperties": false,
"properties": {
"smtp-server": {
"description": "SMTP server used by grafana to send emails (in host:port format). Leaving this empty will disable email sending.",
"type": "string"
},
"smtp-username": {
"description": "Username to connect to SMTP server",
"type": "string"
},
"smtp-password": {
"description": "Password to connect to SMTP server",
"type": "string"
},
"smtp-verify-ssl": {
"description": "Verify SSL certificate of SMTP server",
"type": "boolean",
"default": true
},
"email-from-address": {
"description": "Email address used in From: header of emails",
"type": "string"
},
"email-from-name": {
"description": "Name used in From: header of emails",
"default": "Grafana",
"type": "string"
},
"promtail-extra-scrape-config": {
"description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)",
"default": "",
"type": "string"
}
}
}
instance-output-schema.json 0000664 0000000 0000000 00000002507 13712426147 0032032 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Grafana instantiation",
"additionalProperties": false,
"properties": {
"url": {
"description": "Shared frontend for this Grafana instance",
"pattern": "^https://",
"type": "string"
},
"grafana-username": {
"description": "Admin user for grafana",
"type": "string"
},
"grafana-password": {
"description": "Password for grafana's admin user",
"type": "string"
},
"grafana-url": {
"description": "IPv6 URL to access grafana",
"pattern": "^https://",
"type": "string"
},
"influxdb-url": {
"description": "IPv6 URL of influxdb HTTP endpoint",
"pattern": "^https://",
"type": "string"
},
"influxdb-database": {
"description": "database created in influxdb",
"type": "string"
},
"influxdb-username": {
"description": "username for influxdb",
"type": "string"
},
"influxdb-password": {
"description": "password for influxdb user",
"type": "string"
},
"telegraf-extra-config-dir": {
"description": "Directory in telegraf partition where extra configuration file will be loaded. These files must match *.conf pattern",
"type": "string"
}
},
"type": "object"
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/instance.cfg.in 0000664 0000000 0000000 00000021465 13712426147 0027514 0 ustar 00root root 0000000 0000000 [buildout]
parts =
promises
publish-connection-parameter
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# apache-frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
service = ${:etc}/service
promise = ${:etc}/promise
influxdb-data-dir = ${:srv}/influxdb
grafana-dir = ${:srv}/grafana
grafana-data-dir = ${:grafana-dir}/data
grafana-logs-dir = ${:var}/log
grafana-plugins-dir = ${:grafana-dir}/plugins
grafana-provisioning-config-dir = ${:grafana-dir}/provisioning-config
grafana-provisioning-datasources-dir = ${:grafana-provisioning-config-dir}/datasources
grafana-provisioning-dashboards-dir = ${:grafana-provisioning-config-dir}/dashboards
telegraf-dir = ${:srv}/telegraf
telegraf-extra-config-dir = ${:telegraf-dir}/extra-config
loki-dir = ${:srv}/loki
loki-storage-boltdb-dir = ${:loki-dir}/index/
loki-storage-filesystem-dir = ${:loki-dir}/chunks/
promtail-dir = ${:srv}/promtail
# macros
[generate-certificate]
recipe = plone.recipe.command
command =
if [ ! -e ${:key-file} ]
then
{{ openssl_bin }} req -x509 -nodes -days 3650 \
-subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
-newkey rsa:1024 -keyout ${:key-file} \
-out ${:cert-file}
fi
update-command = ${:command}
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
[config-file]
recipe = slapos.recipe.template:jinja2
template = {{ buildout['parts-directory'] }}/${:_buildout_section_name_}/${:_buildout_section_name_}.cfg.in
rendered = ${directory:etc}/${:_buildout_section_name_}.cfg
mode = 0644
extensions = jinja2.ext.do
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
[influxdb]
ipv6 = ${instance-parameter:ipv6-random}
ipv4 = ${instance-parameter:ipv4-random}
host = ${:ipv6}
local-host = ${:ipv4}
rpc-port = 8088
http-port = 8086
url = https://[${:host}]:${:http-port}
data-dir = ${directory:influxdb-data-dir}
auth-username = ${influxdb-password:username}
auth-password = ${influxdb-password:passwd}
unix-socket = ${directory:var}/influxdb.socket
ssl-cert-file = ${influxdb-certificate:cert-file}
ssl-key-file = ${influxdb-certificate:key-file}
database = telegraf
recipe = slapos.cookbook:wrapper
command-line =
nice -19 chrt --idle 0 ionice -c3 {{ influxd_bin }} -config ${influxdb-config-file:rendered}
wrapper-path = ${directory:service}/influxdb
[influxdb-config-file]
<= config-file
context =
section influxdb influxdb
[influxdb-password]
recipe = slapos.cookbook:generate.password
username = influxdb
[influxdb-certificate]
<= generate-certificate
[influxdb-listen-promise]
<= check-port-listening-promise
hostname = ${influxdb:ipv6}
port = ${influxdb:http-port}
[influxdb-password-promise]
recipe = slapos.cookbook:wrapper
command-line =
{{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
wrapper-path = ${directory:promise}/${:_buildout_section_name_}
[influxdb-database-ready-promise]
recipe = slapos.cookbook:wrapper
command-line =
bash -c "{{ influx_bin }} \
-username ${influxdb:auth-username} \
-password ${influxdb:auth-password} \
-host ${influxdb:host} \
-port ${influxdb:http-port} \
-unsafeSsl \
-ssl \
-execute 'show databases' | grep '${influxdb:database}'"
wrapper-path = ${directory:promise}/${:_buildout_section_name_}
[grafana]
ipv6 = ${instance-parameter:ipv6-random}
port = 8180
url = https://[${:ipv6}]:${:port}
data-dir = ${directory:grafana-data-dir}
logs-dir = ${directory:grafana-logs-dir}
plugins-dir = ${directory:grafana-plugins-dir}
provisioning-config-dir = ${directory:grafana-provisioning-config-dir}
provisioning-datasources-dir = ${directory:grafana-provisioning-datasources-dir}
admin-user = ${grafana-password:username}
admin-password = ${grafana-password:passwd}
secret-key = ${grafana-secret-key:passwd}
ssl-key-file = ${grafana-certificate:key-file}
ssl-cert-file = ${grafana-certificate:cert-file}
recipe = slapos.cookbook:wrapper
command-line =
{{ grafana_bin }} -config ${grafana-config-file:rendered} -homepath {{ grafana_homepath }}
wrapper-path = ${directory:service}/grafana
[grafana-certificate]
<= generate-certificate
[grafana-password]
recipe = slapos.cookbook:generate.password
username = admin
[grafana-secret-key]
recipe = slapos.cookbook:generate.password
[grafana-config-file]
<= config-file
context =
section grafana grafana
section apache_frontend apache-frontend
key slapparameter_dict slap-configuration:configuration
depends =
${grafana-provisioning-config-file:rendered}
[grafana-provisioning-config-file]
<= config-file
rendered = ${grafana:provisioning-datasources-dir}/datasource.yaml
context =
section influxdb influxdb
section loki loki
[grafana-listen-promise]
<= check-port-listening-promise
hostname= ${grafana:ipv6}
port = ${grafana:port}
[telegraf]
recipe = slapos.cookbook:wrapper
extra-config-dir = ${directory:telegraf-extra-config-dir}
# telegraf needs influxdb to be already listening before starting
command-line =
bash -c '${influxdb-listen-promise:path} && nice -19 chrt --idle 0 ionice -c3 {{ telegraf_bin }} --config ${telegraf-config-file:rendered} --config-directory ${:extra-config-dir}'
wrapper-path = ${directory:service}/telegraf
[telegraf-config-file]
<= config-file
context =
section influxdb influxdb
section telegraf telegraf
[loki]
recipe = slapos.cookbook:wrapper
command-line =
bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ loki_bin }} -config.file=${loki-config-file:rendered}'
wrapper-path = ${directory:service}/loki
storage-boltdb-dir = ${directory:loki-storage-boltdb-dir}
storage-filesystem-dir = ${directory:loki-storage-filesystem-dir}
ip = ${instance-parameter:ipv4-random}
port = 3100
url = http://${:ip}:${:port}
[loki-config-file]
<= config-file
context =
section loki loki
[loki-listen-promise]
<= check-url-available-promise
url = ${loki:url}/ready
[promtail]
recipe = slapos.cookbook:wrapper
command-line =
bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ promtail_bin }} -config.file=${promtail-config-file:rendered}'
wrapper-path = ${directory:service}/promtail
dir = ${directory:promtail-dir}
http_port = 19080
ip = ${instance-parameter:ipv4-random}
url = http://${:ip}:${:http_port}
[promtail-config-file]
<= config-file
context =
section promtail promtail
section loki loki
key slapparameter_dict slap-configuration:configuration
[promtail-listen-promise]
<= check-port-listening-promise
hostname= ${promtail:ip}
port = ${promtail:http_port}
[apache-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Grafana Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${grafana:url}
config-https-only = true
return = domain secure_access
[apache-frontend-available-promise]
<= check-url-available-promise
url = ${apache-frontend:connection-secure_access}
[promises]
recipe =
instance-promises =
${influxdb-listen-promise:path}
${influxdb-password-promise:wrapper-path}
${influxdb-database-ready-promise:wrapper-path}
${grafana-listen-promise:path}
${loki-listen-promise:path}
${promtail-listen-promise:path}
${promtail-listen-promise:path}
${apache-frontend-available-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
influxdb-url = ${influxdb:url}
influxdb-database = ${influxdb:database}
influxdb-username = ${influxdb:auth-username}
influxdb-password = ${influxdb:auth-password}
telegraf-extra-config-dir = ${telegraf:extra-config-dir}
grafana-url = ${grafana:url}
grafana-username = ${grafana:admin-user}
grafana-password = ${grafana:admin-password}
loki-url = ${loki:url}
promtail-url = ${promtail:url}
url = ${apache-frontend:connection-secure_access}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/loki-config-file.cfg.in 0000664 0000000 0000000 00000002075 13712426147 0031022 0 ustar 00root root 0000000 0000000 auth_enabled: false
server:
http_listen_port: {{ loki['port'] }}
ingester:
lifecycler:
address: {{ loki['ip'] }}
ring:
kvstore:
store: inmemory
replication_factor: 1
chunk_idle_period: 15m
schema_config:
configs:
- from: 2018-04-15
store: boltdb
object_store: filesystem
schema: v9
index:
prefix: index_
period: 168h
storage_config:
boltdb:
directory: {{ loki['storage-boltdb-dir'] }}
filesystem:
directory: {{ loki['storage-filesystem-dir'] }}
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0
table_manager:
chunk_tables_provisioning:
inactive_read_throughput: 0
inactive_write_throughput: 0
provisioned_read_throughput: 0
provisioned_write_throughput: 0
index_tables_provisioning:
inactive_read_throughput: 0
inactive_write_throughput: 0
provisioned_read_throughput: 0
provisioned_write_throughput: 0
retention_deletes_enabled: false
retention_period: 0
promtail-config-file.cfg.in 0000664 0000000 0000000 00000001135 13712426147 0031630 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana # https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md
server:
http_listen_port: {{ promtail['http_port'] }}
# XXX this external_url does not work ... promtail still listen on all IPs
external_url: {{ promtail['url'] }}
grpc_listen_port: 0
positions:
filename: {{ promtail['dir'] }}/positions.yaml
clients:
- url: {{ loki['url'] }}/api/prom/push
scrape_configs:
- job_name: test
static_configs:
- targets:
- localhost
labels:
job: grafanalogs
__path__: ./var/log/*log
{{ slapparameter_dict.get('promtail-extra-scrape-config', '') }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/software.cfg 0000664 0000000 0000000 00000010355 13712426147 0027131 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../component/make/buildout.cfg
../../component/golang/buildout.cfg
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
buildout.hash.cfg
gowork.cfg
versions = versions
parts =
slapos-cookbook
instance-profile
gowork
influxdb-config-file
telegraf-config-file
grafana-config-file
grafana-provisioning-config-file
loki-config-file
promtail-config-file
[nodejs]
<= nodejs-10.6.0
[yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly,
# otherwise it uses system's nodejs
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz
md5sum = 46790033c23803387890f545e4040690
[gowork]
# All the softwares installed in the go workspace have "non standard" installation
# methods, so we install them in specific parts with custom commands.
# They will be installed because they are dependencies of ${gowork.goinstall}
install =
telegraf-bin = ${:bin}/telegraf
influx-bin = ${:bin}/influx
influxd-bin = ${:bin}/influxd
grafana-bin = ${:bin}/grafana-server
grafana-homepath = ${go_github.com_grafana_grafana:location}
loki-bin = ${:bin}/loki
promtail-bin = ${:bin}/promtail
# use recent go
golang = ${golang1.12:location}
[gowork.goinstall]
command = :
depends =
${influxdb-install:recipe}
${telegraf-install:recipe}
${grafana-install:recipe}
${loki-install:recipe}
${promtail-install:recipe}
[influxdb-install]
<= gowork.goinstall
command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
dep ensure && \
go install ./cmd/..."
update-command =
[telegraf-install]
<= gowork.goinstall
command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/telegraf && \
${make:location}/bin/make &&
cp telegraf ${gowork:bin}"
update-command =
[grafana-install]
<= gowork.goinstall
# yarn and go run build.go needs our nodejs in $PATH
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
cd ${gowork:directory}/src/github.com/grafana/grafana && \
go run build.go setup && \
go run build.go build && \
${yarn:location}/bin/yarn install --pure-lockfile && \
${nodejs:location}/bin/npm run build"
update-command =
[loki-install]
<= gowork.goinstall
# loki also uses nodejs
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/grafana/loki && \
go install ./cmd/loki"
update-command =
[promtail-install]
<= gowork.goinstall
# CGO_ENABLED is to disable systemd support (did not compile in my case)
command = bash -c "export CGO_ENABLED=0 && \
. ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/grafana/loki && \
go install ./cmd/promtail"
update-command =
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[influxdb-config-file]
<= download-file-base
[telegraf-config-file]
<= download-file-base
[grafana-config-file]
<= download-file-base
[grafana-provisioning-config-file]
<= download-file-base
[loki-config-file]
<= download-file-base
[promtail-config-file]
<= download-file-base
[instance-profile]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/instance.cfg
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
key openssl_bin openssl-output:openssl
key telegraf_bin gowork:telegraf-bin
key influxd_bin gowork:influxd-bin
key influx_bin gowork:influx-bin
key grafana_bin gowork:grafana-bin
key grafana_homepath gowork:grafana-homepath
key loki_bin gowork:loki-bin
key promtail_bin gowork:promtail-bin
key curl_bin :curl-bin
key dash_bin :dash-bin
curl-bin = ${curl:location}/bin/curl
dash-bin = ${dash:location}/bin/dash
[versions]
slapos.recipe.template = 4.2
inotifyx = 0.2.2
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/software.cfg.json 0000664 0000000 0000000 00000000562 13712426147 0030100 0 ustar 00root root 0000000 0000000 {
"name": "Grafana",
"description": "Grafana, Telegraf and Influxdb",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"description": "Grafana, Telegraf and Influxdb in same partition",
"request": "instance-input-schema.json",
"response": "instance-output-schema.json",
"index": 0
}
}
}
telegraf-config-file.cfg.in 0000664 0000000 0000000 00000011456 13712426147 0031601 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana # Telegraf configuration
# Telegraf is entirely plugin driven. All metrics are gathered from the
# declared plugins.
# Even if a plugin has no configuration, it must be declared in here
# to be active. Declaring a plugin means just specifying the name
# as a section with no variables. To deactivate a plugin, comment
# out the name and any variables.
# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
# file would generate.
# One rule that plugins conform to is wherever a connection string
# can be passed, the values '' and 'localhost' are treated specially.
# They indicate to the plugin to use their own builtin configuration to
# connect to the local system.
# NOTE: The configuration has a few required parameters. They are marked
# with 'required'. Be sure to edit those to make this configuration work.
# Tags can also be specified via a normal map, but only one form at a time:
[tags]
# dc = "us-east-1"
# Configuration for telegraf agent
[agent]
# Default data collection interval for all plugins
interval = "10s"
# Rounds collection interval to 'interval'
# ie, if interval="10s" then always collect on :00, :10, :20, etc.
round_interval = true
# Default data flushing interval for all outputs. You should not set this below
# interval. Maximum flush_interval will be flush_interval + flush_jitter
flush_interval = "10s"
# Jitter the flush interval by a random amount. This is primarily to avoid
# large write spikes for users running a large number of telegraf instances.
# ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
flush_jitter = "0s"
# Run telegraf in debug mode
debug = false
# Override default hostname, if empty use os.Hostname()
hostname = ""
###############################################################################
# OUTPUTS #
###############################################################################
[outputs]
# Configuration for influxdb server to send metrics to
[outputs.influxdb]
# The full HTTP or UDP endpoint URL for your InfluxDB instance
# Multiple urls can be specified for InfluxDB cluster support.
# urls = ["udp://localhost:8089"] # UDP endpoint example
# XXX XXX XXX
#urls = ["http://localhost:8086"] # required
urls = ["{{ influxdb['url'] }}"]
insecure_skip_verify = true # because we are using a self signed certificate
# The target database for metrics (telegraf will create it if not exists)
database = "{{ influxdb['database'] }}" # required
# Precision of writes, valid values are n, u, ms, s, m, and h
# note: using second precision greatly helps InfluxDB compression
precision = "s"
# Connection timeout (for the connection with InfluxDB), formatted as a string.
# If not provided, will default to 0 (no timeout)
# timeout = "5s"
username = "{{ influxdb['auth-username'] }}"
password = "{{ influxdb['auth-password'] }}"
# Set the user agent for HTTP POSTs (can be useful for log differentiation)
# user_agent = "telegraf"
# Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
# udp_payload = 512
###############################################################################
# PLUGINS #
###############################################################################
# Read metrics about cpu usage
[cpu]
# Whether to report per-cpu stats or not
percpu = true
# Whether to report total system cpu stats or not
totalcpu = true
# Comment this line if you want the raw CPU time metrics
drop = ["cpu_time"]
# Read metrics about memory usage
[mem]
# no configuration
[disk]
[io]
[system]
###############################################################################
# ERP5 - PLUGINS #
###############################################################################
#
# Left here as example, don't edit this file directly, but place your config
# files in {{ telegraf['extra-config-dir'] }}
#
#[mysql]
# servers = ["root@unix(/srv/slapgrid/slappart12/srv/runner/instance/slappart1/var/run/mariadb.sock)/erp5"]
#[memcached]
# # XXX kumofs does not support memcached's stat command
# servers = ["10.0.248.233:2013", "10.0.248.233:2003"]
#[haproxy]
# servers = ["http://10.0.121.162:2150/haproxy", "http://10.0.121.162:2152/haproxy"]
#[[inputs.exec]]
# commands = ["/srv/slapgrid/slappart0/bin/slapsensor /srv/slapgrid/slappart0/srv/runner/instance/etc/supervisord.conf"]
# name_suffix = "_slapos"
# interval = "5s"
###############################################################################
# SERVICE PLUGINS #
###############################################################################
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/test/ 0000775 0000000 0000000 00000000000 13712426147 0025571 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/test/README.md 0000664 0000000 0000000 00000000043 13712426147 0027045 0 ustar 00root root 0000000 0000000 Tests for Grafana software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/test/setup.py 0000664 0000000 0000000 00000003703 13712426147 0027306 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.grafana'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Grafana",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
'supervisor',
'psutil',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/grafana/test/test.py 0000664 0000000 0000000 00000020142 13712426147 0027121 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import textwrap
import logging
import tempfile
import time
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class GrafanaTestCase(SlapOSInstanceTestCase):
"""Base test case for grafana.
Since the instances takes timte to start and stop,
we increate as lot the number of retries.
"""
report_max_retry = 30
instance_max_retry = 30
class TestGrafana(GrafanaTestCase):
def setUp(self):
self.grafana_url = self.computer_partition.getConnectionParameterDict(
)['grafana-url']
def test_grafana_available(self):
resp = requests.get(self.grafana_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
def test_grafana_api(self):
# check API is usable
api_org_url = '{self.grafana_url}/api/org'.format(**locals())
resp = requests.get(api_org_url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
connection_params = self.computer_partition.getConnectionParameterDict()
resp = requests.get(
api_org_url,
verify=False,
auth=requests.auth.HTTPBasicAuth(
connection_params['grafana-username'],
connection_params['grafana-password'],
))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertEqual(1, resp.json()['id'])
def test_grafana_datasource_povisinonned(self):
# data sources are provisionned
connection_params = self.computer_partition.getConnectionParameterDict()
resp = requests.get(
'{self.grafana_url}/api/datasources'.format(**locals()),
verify=False,
auth=requests.auth.HTTPBasicAuth(
connection_params['grafana-username'],
connection_params['grafana-password'],
))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertEqual(
sorted(['influxdb', 'loki']),
sorted([ds['type'] for ds in resp.json()]))
class TestInfluxDb(GrafanaTestCase):
def setUp(self):
self.influxdb_url = self.computer_partition.getConnectionParameterDict(
)['influxdb-url']
def test_influxdb_available(self):
ping_url = '{self.influxdb_url}/ping'.format(**locals())
resp = requests.get(ping_url, verify=False)
self.assertEqual(requests.codes.no_content, resp.status_code)
def test_influxdb_api(self):
query_url = '{self.influxdb_url}/query'.format(**locals())
connection_params = self.computer_partition.getConnectionParameterDict()
for i in range(10):
# retry, as it may take a little delay to create databases
resp = requests.get(
query_url,
verify=False,
params=dict(
q='SHOW DATABASES',
u=connection_params['influxdb-username'],
p=connection_params['influxdb-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
result, = resp.json()['results']
if result['series'] and 'values' in result['series'][0]:
break
time.sleep(0.5 * i)
self.assertIn(
[connection_params['influxdb-database']], result['series'][0]['values'])
class TestTelegraf(GrafanaTestCase):
def test_telegraf_running(self):
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if 'telegraf' in p['name']]
self.assertEqual('RUNNING', process_info['statename'])
class TestLoki(GrafanaTestCase):
@classmethod
def getInstanceParameterDict(cls):
cls._logfile = tempfile.NamedTemporaryFile(suffix='log')
return {
'promtail-extra-scrape-config':
textwrap.dedent(
r'''
- job_name: {cls.__name__}
pipeline_stages:
- regex:
expression: "^(?P.*) - (?P\\S+) - (?P\\S+) - (?P.*)"
- timestamp:
format: 2006-01-02T15:04:05Z00:00
source: timestamp
- labels:
level:
name:
static_configs:
- targets:
- localhost
labels:
job: {cls.__name__}
__path__: {cls._logfile.name}
''').format(**locals())
}
@classmethod
def tearDownClass(cls):
cls._logfile.close()
super(TestLoki, cls).tearDownClass()
def setUp(self):
self.loki_url = self.computer_partition.getConnectionParameterDict(
)['loki-url']
def test_loki_available(self):
self.assertEqual(
requests.codes.ok,
requests.get('{self.loki_url}/ready'.format(**locals()),
verify=False).status_code)
def test_log_ingested(self):
# create a logger logging to the file that we have
# configured in instance parameter.
test_logger = logging.getLogger(self.id())
test_logger.setLevel(logging.INFO)
test_handler = logging.FileHandler(filename=self._logfile.name)
test_handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
test_logger.addHandler(test_handler)
test_logger.info("testing message")
test_logger.info("testing another message")
test_logger.warning("testing warn")
# Check our messages have been ingested
# we retry a few times, because there's a short delay until messages are
# ingested and returned.
for i in range(10):
resp = requests.get(
'{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
**locals()),
verify=False).json()
if not resp:
time.sleep(0.5 * i)
continue
warn_stream_list = [stream for stream in resp['streams'] if 'level="WARNING"' in stream['labels']]
self.assertEqual(1, len(warn_stream_list), resp['streams'])
warn_stream, = warn_stream_list
self.assertIn("testing warn", warn_stream['entries'][0]['line'])
info_stream_list = [stream for stream in resp['streams'] if 'level="INFO"' in stream['labels']]
self.assertEqual(1, len(info_stream_list), resp['streams'])
info_stream, = info_stream_list
self.assertTrue(
[
line for line in info_stream['entries']
if "testing message" in line['line']
])
self.assertTrue(
[
line for line in info_stream['entries']
if "testing another message" in line['line']
])
# The labels we have configued are also available
resp = requests.get(
'{self.loki_url}/api/prom/label'.format(**locals()),
verify=False).json()
self.assertIn('level', resp['values'])
self.assertIn('name', resp['values'])
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/ 0000775 0000000 0000000 00000000000 13712426147 0025170 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/instance-root.cfg.in 0000664 0000000 0000000 00000003200 13712426147 0031036 0 ustar 00root root 0000000 0000000 {% set sla_dict = {} -%}
{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
{% do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
{% endfor -%}
{% macro sla(name, required=False) -%}
{% if required or name in sla_dict -%}
{% for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
sla-{{ k }} = {{ v }}
{% endfor -%}
{% else -%}
sla-computer_guid = ${slap-connection:computer-id}
{% endif -%}
{% endmacro -%}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
# Executables put here will be started but not monitored (for startup scripts)
script = ${:etc}/run
# Executables put here will be started and monitored (for daemons)
service = ${:etc}/service
[server]
<= request-common-base
software-type = server
name = server
{{ sla('server') }}
return =
[proxy]
recipe = slapos.cookbook:wrapper
command-line = {{ python_executable }} {{ rina_proxy }} ${server:instance-guid} ${:ipv6} ${:port}
wrapper-path = ${directory:service}/proxy
environment =
PATH={{ rina_tools_location }}/bin:%(PATH)s
ipv6 = {{ipv6}}
port = 8080
[publish]
recipe = slapos.cookbook:publish.serialised
url.proxy = http://[${proxy:ipv6}]:${proxy:port}
[request-common-base]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[buildout]
parts =
publish
{{- assert(not sla_dict, sla_dict) }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/instance-server.cfg.in 0000664 0000000 0000000 00000000551 13712426147 0031367 0 ustar 00root root 0000000 0000000 [directory]
recipe = slapos.cookbook:mkdirectory
service = ${buildout:directory}/etc/service
[server]
recipe = slapos.cookbook:wrapper
command-line = {{rina_tools_location}}/bin/rina-echo-time -l --server-api {{instance_guid}}
wrapper-path = ${directory:service}/server
[publish]
recipe = slapos.cookbook:publish.serialised
[buildout]
parts = publish server
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/instance.cfg.in 0000664 0000000 0000000 00000002532 13712426147 0030064 0 ustar 00root root 0000000 0000000 [buildout]
parts = switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extensions = jinja2.ext.do
extra-context =
context =
key slapparameter_dict slap-configuration:configuration
raw rina_tools_location {{ rina_tools_location }}
${:extra-context}
[root]
<= jinja2-template-base
template = {{ instance_root }}
extra-context =
import urlparse urlparse
key ipv6 slap-configuration:ipv6-random
raw python_executable {{ python_executable }}
raw rina_proxy {{ rina_proxy }}
[server]
<= jinja2-template-base
template = {{ instance_server }}
extra-context =
key instance_guid slap-configuration:instance-guid
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
override = {{ dumps(override_switch_softwaretype |default) }}
default = root:rendered
# BBB
RootSoftwareInstance = ${:default}
server = server:rendered
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/proxy 0000775 0000000 0000000 00000004251 13712426147 0026301 0 ustar 00root root 0000000 0000000 #!/usr/bin/python
import httplib, socket, subprocess, sys, threading, urlparse
import BaseHTTPServer, SocketServer
class Popen(subprocess.Popen):
def stop(self):
if self.pid and self.returncode is None:
self.terminate()
t = threading.Timer(5, self.kill)
t.start()
# PY3: use waitid(WNOWAIT) and call self.poll() after t.cancel()
r = self.wait()
t.cancel()
return r
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
try:
path, query = self.path.split('?', 1)
except ValueError:
path = self.path
query = {}
else:
query = dict(urlparse.parse_qsl(query,
keep_blank_values=1, strict_parsing=1))
_, path = path.split('/')
if not _:
try:
return getattr(self, '_GET_' + path, None)(**query)
except (AttributeError, TypeError):
pass
self.send_error(httplib.BAD_REQUEST)
def _GET_(self):
self.send_response(httplib.FOUND)
self.send_header('Location', 'ping?count=4')
self.end_headers()
def _GET_ping(self, count=4):
count = int(count)
args = "rina-echo-time", "--server-api", server_api, "-c", str(count)
try:
p = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except Exception, e:
self.send_error(httplib.INTERNAL_SERVER_ERROR)
raise
try:
self.send_response(httplib.OK)
self.send_header('Content-type', 'text/plain; charset=utf-8')
self.end_headers()
self.wfile.write('# count=%s\n' % count)
while 1:
line = p.stdout.readline()
if not line:
break
self.wfile.write(line)
finally:
p.stop()
class Server(SocketServer.ThreadingTCPServer):
allow_reuse_address = True
daemon_threads = True
address_family = socket.AF_INET6
if __name__ == "__main__":
server_api = sys.argv[1]
Server((sys.argv[2], int(sys.argv[3])), Handler).serve_forever()
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/hellorina/software.cfg 0000664 0000000 0000000 00000002350 13712426147 0027503 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../component/rina-tools/buildout.cfg
parts =
slapos-cookbook
template
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = f6c61225990986d94d0017b07b873aa7
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key python_executable python:executable
key rina_tools_location rina-tools:location
key instance_root instance-root:target
key instance_server instance-server:target
key rina_proxy proxy:target
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
# XXX: following mode should be the default
mode = 644
[instance-root]
<= download-base
md5sum = f647054be67998f9eece174f106c4464
[instance-server]
<= download-base
md5sum = 88a451b0f7f8def12713b92b91659b98
[proxy]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
mode = 755
md5sum = 78b77a6bda9958f547f7d89b747731e3
[versions]
slapos.recipe.template = 4.4
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/ 0000775 0000000 0000000 00000000000 13712426147 0025366 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/buildout.hash.cfg 0000664 0000000 0000000 00000001433 13712426147 0030621 0 ustar 00root root 0000000 0000000 # THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = dcb9d2f540e0e397c9346c8b0c05f233
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/instance.cfg.in 0000664 0000000 0000000 00000011131 13712426147 0030255 0 ustar 00root root 0000000 0000000 #############################
#
# Deploy hello-world instance
#
#############################
[buildout]
parts =
directory
publish-connection-parameter
# Define egg directories to be the one from Software Release
# (/opt/slapgrid/...)
# Always the same.
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
extends = {{ template_monitor }}
[instance-parameter]
# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
# We use the slapconfiguration recipe with a few parameters (partition id,
# computer id, certificate, etc).
# It will then authenticate to SlapOS Master and fetch the instance parameters.
# The parameters are accessible from ${instance-parameter:configuration.name-of-parameter}
# Always the same. Just copy/paste.
# See docstring of slapos.cookbook:slapconfiguration for more information.
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# Define default parameter(s) that will be used later, in case user didn't
# specify it.
# All possible parameters should have a default.
# In our use case, we are expecting from the user to specify one (optional) parameter: "name". We put the default value here if he doesn't specify it, so that it doesn't crash.
configuration.name = John Doe
# If our use case requires that the user can specify a mail address so that his instance can mail to him (for example), we can do:
# configuration.mail-address =
# If the user doesn't specify it, it won't break and the recipe can handle it (i.e don't send any mail for example).
# Create all needed directories, depending on your needs
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
# Executables put here will be started but not monitored (for startup scripts)
script = ${:etc}/run/
# Executables put here will be started and monitored (for daemons)
service = ${:etc}/service
# Path of the log directory used by our service (see [helloweb])
log = ${:var}/log
# Create a simple web server that says "hello " to the web.
[helloweb]
# helloworld service is listening on:
# - global IPv6 address, and
# - fixed port
#
# NOTE because every computer partition is allocated its own global IPv6
# address, it is ok to fix the port - different hello-world instances will have
# different IPv6 addresses and they all will be accessible at the same time.
ipv6 = ${instance-parameter:ipv6-random}
# full URL - for convenience
url = http://[${:ipv6}]:${:port}
# the service will log here
logfile = ${directory:log}/helloweb-${:kind}.log
# Actual script that starts the service:
# This recipe will try to "exec" the command-line after separating parameters.
recipe = slapos.cookbook:wrapper
command-line =
{{ buildout['bin-directory'] }}/helloweb-${:kind} --logfile ${:logfile}
${:ipv6} ${:port} ${instance-parameter:configuration.name}
# Put this shell script in the "etc/service" directory. Each executable of this
# repository will be started and monitored by supervisord. If a service
# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
wrapper-path = ${directory:service}/helloweb-${:kind}
# promise, that checks that helloweb service is alive
[helloweb-promise]
<= monitor-promise-base
module = check_port_listening
name = helloweb-${:kind}.py
{# macro to instantiate service of `kind` to listen on `port` #}
{% set service_list = [] %}
{% macro hellowebsrv(kind, port) %}
{% do service_list.append(kind) %}
[helloweb-{{ kind }}]
<= helloweb
kind = {{ kind }}
port = {{ port }}
[helloweb-{{ kind }}-promise]
<= helloweb-promise
kind = {{ kind }}
config-hostname= ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
{% endmacro %}
# services instantiation
{{ hellowebsrv('python', 7777) }}
{{ hellowebsrv('ruby', 7778) }}
{{ hellowebsrv('go', 7779) }}
# register all services/promises to buildout parts
[buildout]
parts +=
{%- for kind in service_list %}
helloweb-{{ kind }}
helloweb-{{ kind }}-promise
{%- endfor %}
# Publish all the parameters needed for the user to connect to the instance.
# It can be anything: URL(s), password(s), or arbitrary parameters.
# Here we'll just echo back the entered name as instance parameter
[publish-connection-parameter]
recipe = slapos.cookbook:publish
name = Hello ${instance-parameter:configuration.name}!
{%- for kind in service_list %}
url.{{ kind }} = ${helloweb-{{ kind }}:url}
{%- endfor %}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/software.cfg 0000664 0000000 0000000 00000003601 13712426147 0027701 0 ustar 00root root 0000000 0000000 [buildout]
extends =
# buildout.hash.cfg is used for automated hash calculation of managed
# instance files by calling update-hash
buildout.hash.cfg
# "slapos" stack describes basic things needed for 99.9% of SlapOS Software
# Releases
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
# Extend here component profiles, like openssl, apache, mariadb, curl...
# Or/and extend a stack (lamp, tomcat) that does most of the work for you
# In this example we extend from helloweb component.
# ../../component/component1/buildout.cfg
# ../../component/component2/buildout.cfg
../../component/helloweb/buildout.cfg
parts =
# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
# in 99,9% of Slapos Software Releases)
slapos-cookbook
# Call creation of instance.cfg file that will be called for deployment of
# instance
instance-profile
# build helloweb programs
helloweb-python
helloweb-ruby
helloweb-go
# Macro for jinja templates. The filename is set in buildout.hash.cfg
# in the section using this template
[jinja-template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
mode = 0644
# Download instance.cfg.in (buildout profile used to deployment of instance),
# replace all ${foo:bar} parameters by real values, and change $${foo:bar} to
# ${foo:bar}
# The recipe, template and mode are fetched from jijna-template
[instance-profile]
# The <= is buildout verb to use defined template
<= jinja-template
rendered = ${buildout:directory}/instance.cfg
extensions = jinja2.ext.do
context =
section buildout buildout
raw template_monitor ${monitor2-template:rendered}
# md5sum is fetched from buildout.hash.cfg and can be recalculated automatically by
# calling update-hash
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/test/ 0000775 0000000 0000000 00000000000 13712426147 0026345 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/test/README.md 0000664 0000000 0000000 00000000046 13712426147 0027624 0 ustar 00root root 0000000 0000000 Tests for HelloWorld software release
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/test/setup.py 0000664 0000000 0000000 00000003660 13712426147 0030064 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.helloworld'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' helloworld",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
],
zip_safe=True,
test_suite='test',
)
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/helloworld/test/test.py 0000664 0000000 0000000 00000004653 13712426147 0027706 0 ustar 00root root 0000000 0000000 ##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class HelloWorldTestCase(SlapOSInstanceTestCase):
# to be defined by subclasses
name = None
kind = None
@classmethod
def getInstanceParameterDict(cls):
return {"name": cls.name}
class HTTPRequestTestMixin(object):
"""Test that the service url.${kind} responds Hello ${name}
"""
def test_get(self):
url = self.computer_partition.getConnectionParameterDict()['url.{}'.format(
self.kind)]
response = requests.get(url)
self.assertEqual(requests.codes['OK'], response.status_code)
self.assertTrue(
response.text.startswith("Hello {}".format(self.name)), response.text)
class TestPython(HelloWorldTestCase, HTTPRequestTestMixin):
name = "Python"
kind = "python"
class TestRuby(HelloWorldTestCase, HTTPRequestTestMixin):
name = "Ruby"
kind = "ruby"
class TestGolang(HelloWorldTestCase, HTTPRequestTestMixin):
name = "Go"
kind = "go"
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/ 0000775 0000000 0000000 00000000000 13712426147 0024570 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/README.md 0000664 0000000 0000000 00000000746 13712426147 0026056 0 ustar 00root root 0000000 0000000 # Html5 Application Server #
## Presentation ##
* Fast hosting software for static website (html5)
* Use Nginx server
## Parameter ##
download_url (string) :required
Details :
* Only tarball (tar) is supported
* Compressed format is gunzip (optional)
* Tarball must contain an index.html at its root
## How it works ##
Each time you (re)start your instance or update parameters, html5as will remove previous website then download tarball and extract it in docroot directory.
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/instance.cfg 0000664 0000000 0000000 00000000403 13712426147 0027052 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance_html5as:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/instance_html5as.cfg 0000664 0000000 0000000 00000006153 13712426147 0030517 0 ustar 00root root 0000000 0000000 [buildout]
parts =
nginx_conf
downloader
launcher
mime_types
publish-connection-information
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
# partition tree
# /
# |- etc/
# | |- nginx.conf
# | |- run/
# | |- html5as (binary)
# |- var/
# | |- run/
# | | |- nginx.pid
# | |- log/
# | | |- nginx.log
# | | |- nginx.access.log
# |- srv/
# | |- html5as/ (doc root)
# | | |- index.html
# | |- backup/
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
srv = $${buildout:directory}/srv
tmp = $${buildout:directory}/tmp
[basedirectory]
recipe = slapos.cookbook:mkdirectory
services = $${rootdirectory:etc}/run
log = $${rootdirectory:var}/log
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
data = $${rootdirectory:srv}/html5as
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
client_body_temp_path = $${rootdirectory:tmp}/client_body_temp_path
proxy_temp_path = $${rootdirectory:tmp}/proxy_temp_path
fastcgi_temp_path = $${rootdirectory:tmp}/fastcgi_temp_path
uwsgi_temp_path = $${rootdirectory:tmp}/uwsgi_temp_path
scgi_temp_path = $${rootdirectory:tmp}/scgi_temp_path
[html5as]
# Options
nb_workers = 2
# Network
ip = $${slap-network-information:global-ipv6}
port = 8081
# Paths
# Log
path_pid = $${basedirectory:run}/nginx.pid
path_log = $${basedirectory:log}/nginx.log
path_access_log = $${basedirectory:log}/nginx.access.log
path_error_log = $${basedirectory:log}/nginx.error.log
path_tmp = $${buildout:directory}/tmp
# Docroot
docroot = $${basedirectory:data}
default_index = $${basedirectory:data}/index.html
# Config files
path_nginx_conf = $${rootdirectory:etc}/nginx.conf
path_mime_types = $${rootdirectory:etc}/mime_types
# Executables
bin_nginx = ${nginx:location}/sbin/nginx
bin_launcher = $${basedirectory:services}/launcher
bin_downloader = $${basedirectory:services}/downloader
# Utils
path_shell = ${dash:location}/bin/dash
path_curl = ${curl:location}/bin/curl
path_tar = ${tar:location}/bin/tar
[nginx_conf]
recipe = slapos.recipe.template:jinja2
template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
rendered = $${html5as:path_nginx_conf}
context =
section param_html5as html5as
section param_tempdir tempdirectory
[mime_types]
recipe = slapos.recipe.template:jinja2
template = ${template_mime_types:location}/${template_mime_types:filename}
rendered = $${html5as:path_mime_types}
[downloader]
recipe = slapos.recipe.template:jinja2
template = ${template_downloader:location}/${template_downloader:filename}
rendered = $${html5as:bin_downloader}
mode = 700
context =
section param_html5as html5as
key download_url slap-parameter:download_url
[launcher]
recipe = slapos.recipe.template:jinja2
template = ${template_launcher:location}/${template_launcher:filename}
rendered = $${html5as:bin_launcher}
mode = 700
context =
section param_html5as html5as
[publish-connection-information]
recipe = slapos.cookbook:publish
server_url = http://[$${html5as:ip}]:$${html5as:port}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/software.cfg 0000664 0000000 0000000 00000003723 13712426147 0027110 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../component/nginx/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/tar/buildout.cfg
parts =
dash
tar
curl
nginx
slapos-cookbook
template
template_nginx_conf
template_downloader
template_launcher
template_mime_types
instance_html5as
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 918e0d6513e4d1c92051431d83261dab
mode = 0644
[instance_html5as]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance_html5as.cfg
output = ${buildout:directory}/template_html5as.cfg
md5sum = 41cb6178f760238ca276854873ef9364
mode = 0644
[template_nginx_conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/nginx_conf.in
md5sum = 61dc4c82bf48563228ce4dea6c5c6319
filename = nginx_conf.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template_launcher]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/launcher.in
md5sum = acf5bb55ceac2e826259d28ed5c1de3a
filename = launcher.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template_downloader]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/downloader.in
md5sum = 9779e2db6c73d282f802b3407b390ede
filename = downloader.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template_mime_types]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/mime_types.in
md5sum = 4ef94a7b458d885cd79ba0b930a5727e
filename = mime_types.in
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[versions]
async = 0.6.1
gitdb = 0.5.4
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.4
smmap = 0.8.2
plone.recipe.command = 1.1
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/templates/ 0000775 0000000 0000000 00000000000 13712426147 0026566 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/templates/downloader.in 0000664 0000000 0000000 00000000611 13712426147 0031252 0 ustar 00root root 0000000 0000000 #! {{ param_html5as['path_shell'] }}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
# Remove exsistant website
rm -r {{ param_html5as['docroot'] }}/*
# Download and extract website tarball into docroot directory
exec {{ param_html5as['path_curl'] }} -ks {{ download_url }} | {{ param_html5as['path_tar'] }} -z -x -C {{ param_html5as['docroot'] }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/templates/launcher.in 0000664 0000000 0000000 00000000335 13712426147 0030720 0 ustar 00root root 0000000 0000000 #! {{ param_html5as['path_shell'] }}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
# Run nginx
exec {{ param_html5as['bin_nginx'] }} -c {{ param_html5as['path_nginx_conf'] }}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/templates/mime_types.in 0000664 0000000 0000000 00000004013 13712426147 0031267 0 ustar 00root root 0000000 0000000 types {
text/html html htm shtml;
text/css css;
text/xml xml rss;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
application/ogg ogx;
audio/midi mid midi kar;
audio/mpeg mpga mpega mp2 mp3 m4a;
audio/ogg oga ogg spx;
audio/x-realaudio ra;
audio/webm weba;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg mpe;
video/ogg ogv;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5as/templates/nginx_conf.in 0000664 0000000 0000000 00000001720 13712426147 0031246 0 ustar 00root root 0000000 0000000 worker_processes {{ param_html5as['nb_workers'] }};
pid {{ param_html5as['path_pid'] }};
error_log {{ param_html5as['path_error_log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
include {{ param_html5as['path_mime_types'] }};
default_type application/octet-stream;
types_hash_bucket_size 64;
access_log {{ param_html5as['path_access_log'] }} combined;
index index.html;
server {
listen [{{ param_html5as['ip'] }}]:{{ param_html5as['port'] }};
server_name _;
keepalive_timeout 5;
client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
# path for static files
root {{ param_html5as['docroot'] }};
}
}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/ 0000775 0000000 0000000 00000000000 13712426147 0024726 5 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/instance-html5ide.cfg 0000664 0000000 0000000 00000011577 13712426147 0030737 0 ustar 00root root 0000000 0000000 [buildout]
parts =
httpd-launcher
httpd-graceful
httpd-dav-htaccess
httpd-make-dummy-index-html
promise-httpd-ipv6
publish-connection-information
## Monitoring part
monitor-base
extends = ${monitor-template:output}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
service = $${:etc}/service/
etc-run = $${:etc}/run/
log = $${:var}/log/
run = $${:var}/run/
backup = $${:srv}/backup/
promise = $${:etc}/promise/
# Jinja Common
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key slap_software_type instance-parameter:slap-software-type
key slapparameter_dict instance-parameter:configuration
section directory directory
$${:extra-context}
###########################
# Apache configuration
#
[apache-directory]
recipe = slapos.cookbook:mkdirectory
www = $${directory:srv}/www
dav = $${directory:srv}/dav
slave-configuration = $${directory:etc}/apache-slave-conf.d/
ssl = $${directory:etc}/ssl
[apache-configuration]
httpd-launcher = $${directory:service}/httpd
httpd-graceful = $${directory:etc-run}/httpd-graceful
frontend-configuration = $${directory:etc}/httpd.conf
access-log = $${directory:log}/httpd-access.log
error-log = $${directory:log}/httpd-error.log
pid-file = $${directory:run}/httpd.pid
httpd-graceful-command = kill -USR1 $(cat $${:pid-file})
# Configuration
[httpd-parameter]
path_pid = $${directory:run}/httpd.pid
global_ip = $${slap-network-information:global-ipv6}
global_port = 6666
path_error_log = $${apache-configuration:error-log}
path_access_log = $${apache-configuration:access-log}
key_file = $${ca-httpd:key-file}
cert_file = $${ca-httpd:cert-file}
document_root = $${apache-directory:www}
dav_lock = $${directory:var}/DavLock
dav_home = $${apache-directory:dav}
htpassword_path = $${directory:etc}/.htpasswd
etc_dir = $${directory:etc}
var_dir = $${directory:var}
[httpd-conf]
recipe = slapos.recipe.template:jinja2
template = ${template_httpd_conf:location}/${template_httpd_conf:filename}
rendered = $${apache-configuration:frontend-configuration}
context =
section parameter httpd-parameter
# Htaccess for dav storage
[httpd-dav-htaccess]
recipe = plone.recipe.command
stop-on-error = true
htaccess-path = $${httpd-parameter:htpassword_path}
command = ${apache:location}/bin/htpasswd -cb $${:htaccess-path} $${:user} $${:password}
user = admin
password = admin
# Launcher and Graceful
[httpd-launcher]
recipe = slapos.cookbook:wrapper
apache-executable = ${apache:location}/bin/httpd
wrapper-path = $${apache-configuration:httpd-launcher}
command-line = $${:apache-executable} -f $${httpd-conf:rendered} -DFOREGROUND
[httpd-graceful]
recipe = slapos.recipe.template:jinja2
template = ${template-wrapper:output}
rendered = $${apache-configuration:httpd-graceful}
mode = 0700
context =
key content apache-configuration:httpd-graceful-command
# Create Dummu index.html
[httpd-make-dummy-index-html]
recipe = plone.recipe.command
command = if [ ! -f $${httpd-parameter:document_root}/index.html ]; then echo "Hello World!!!" > $${httpd-parameter:document_root}/index.html ; fi
# Promise
[promise-httpd-ipv6]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promise}/httpd_ipv6
hostname = $${httpd-parameter:global_ip}
port = $${httpd-parameter:global_port}
###########################
# Certificate Authority
#
[cadirectory]
recipe = slapos.cookbook:mkdirectory
ca-dir = $${directory:srv}/ssl
requests = $${:ca-dir}/requests/
private = $${:ca-dir}/private/
certs = $${:ca-dir}/certs/
newcerts = $${:ca-dir}/newcerts/
crl = $${:ca-dir}/crl/
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = ${openssl:location}/bin/openssl
ca-dir = $${cadirectory:ca-dir}
requests-directory = $${cadirectory:requests}
wrapper = $${directory:service}/certificate_authority
ca-private = $${cadirectory:private}
ca-certs = $${cadirectory:certs}
ca-newcerts = $${cadirectory:newcerts}
ca-crl = $${cadirectory:crl}
[ca-httpd]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = $${cadirectory:certs}/httpd.key
cert-file = $${cadirectory:certs}/httpd.crt
executable = $${directory:service}/ca-certificate
wrapper = $${directory:service}/ca-certificate
# Put domain name
name = example.com
###########################
# Monitor Part
#
[publish-connection-information]
recipe = slapos.cookbook:publish
server_url = https://[$${httpd-parameter:global_ip}]:$${httpd-parameter:global_port}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/instance.cfg 0000664 0000000 0000000 00000000405 13712426147 0027212 0 ustar 00root root 0000000 0000000 [buildout]
parts =
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-html5-ide:output}
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/software.cfg 0000664 0000000 0000000 00000002365 13712426147 0027247 0 ustar 00root root 0000000 0000000 [buildout]
extends =
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/apache/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/nodejs/buildout.cfg
parts =
slapos-cookbook
nodejs
template
template_httpd_conf
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 8fd9378f81c26be62e5c93bf52c837a5
output = ${buildout:directory}/template.cfg
mode = 0644
[template-html5-ide]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-html5ide.cfg
md5sum = 21331b97b1fffe3567fdbb079ff147b9
output = ${buildout:directory}/template-html5ide.cfg
mode = 0644
[template_httpd_conf]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/template/httpd.conf.jinja2
download-only = true
md5sum = 97d84138323b1e3214847b1b7de9a10e
filename = httpd_conf.in
mode = 0644
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[versions]
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.4
slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/template/ 0000775 0000000 0000000 00000000000 13712426147 0026541 5 ustar 00root root 0000000 0000000 httpd.conf.jinja2 0000664 0000000 0000000 00000004164 13712426147 0031635 0 ustar 00root root 0000000 0000000 slapos-c6cc45ab7b5d52c653832697d6e0323159c12837-software/software/html5ide/template PidFile "{{ parameter.path_pid }}"
ServerName example.com
ServerAdmin someone@email
Listen [{{ parameter.global_ip }}]:{{ parameter.global_port }}
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule env_module modules/mod_env.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule dir_module modules/mod_dir.so
ErrorLog "{{ parameter.path_error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b" common
CustomLog "{{ parameter.path_access_log }}" common
# SSL Configuration
Define SSLConfigured
SSLCertificateFile {{ parameter.cert_file }}
SSLCertificateKeyFile {{ parameter.key_file }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLEngine On
DavLockDB {{ parameter.dav_lock }}
Alias /share {{ parameter.dav_home }}
DirectoryIndex disabled
DAV On
Options Indexes FollowSymLinks
AuthType Basic
AuthName "webdav"
AuthUserFile "{{ parameter.htpassword_path }}"
Require valid-user
Alias / {{ parameter.document_root }}/
Order Allow,Deny
Allow from all
Satisfy Any
DirectoryIndex index.html
Options FollowSymLinks