Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Justin
slapos
Commits
6f2caaa2
Commit
6f2caaa2
authored
Apr 07, 2021
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Plain Diff
re6stnet: clean-up & backup
See merge request
!940
parents
b4823d48
aad7050a
Changes
10
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
101 additions
and
180 deletions
+101
-180
software/re6stnet/apache.conf.in
software/re6stnet/apache.conf.in
+0
-17
software/re6stnet/buildout.hash.cfg
software/re6stnet/buildout.hash.cfg
+4
-12
software/re6stnet/instance-re6stnet-input-schema.json
software/re6stnet/instance-re6stnet-input-schema.json
+5
-0
software/re6stnet/instance-re6stnet.cfg.in
software/re6stnet/instance-re6stnet.cfg.in
+42
-81
software/re6stnet/instance.cfg.in
software/re6stnet/instance.cfg.in
+5
-9
software/re6stnet/re6st-registry.conf.in
software/re6stnet/re6st-registry.conf.in
+1
-1
software/re6stnet/registry-run.in
software/re6stnet/registry-run.in
+0
-4
software/re6stnet/software.cfg
software/re6stnet/software.cfg
+44
-20
software/re6stnet/test/test.py
software/re6stnet/test/test.py
+0
-33
software/re6stnet/wrapper.in
software/re6stnet/wrapper.in
+0
-3
No files found.
software/re6stnet/apache.conf.in
View file @
6f2caaa2
...
...
@@ -8,7 +8,6 @@ LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so
#LoadModule dav_module modules/mod_dav.so
#LoadModule dav_fs_module modules/mod_dav_fs.so
...
...
@@ -31,17 +30,6 @@ ErrorLog "{{ error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ access_log }}" combined
{% if uri_scheme == 'https' -%}
# SSL Configuration
SSLCertificateFile {{ certificate }}
SSLCertificateKeyFile {{ key }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
{% endif -%}
<Directory />
Options FollowSymLinks
AllowOverride None
...
...
@@ -50,10 +38,5 @@ SSLHonorCipherOrder on
Listen {{ ipv6 }}:{{ apache_port }}
<VirtualHost *:{{ apache_port }}>
{% if uri_scheme == 'https' -%}
SSLEngine On
SSLProxyEngine On
{% endif -%}
ProxyPass / http://{{ re6st_ipv4 }}:{{ re6st_port }}/
</VirtualHost>
software/re6stnet/buildout.hash.cfg
View file @
6f2caaa2
...
...
@@ -14,24 +14,16 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum =
610fc6fd0444d3bab3fca4478572749
a
md5sum =
01938aa1683b7994f814fff6d964b9b
a
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum =
002f7405f565c82219b0d4b92790ff8c
md5sum =
4e4475c8ef80a3a53aa63a2ae444586d
[template-apache-conf]
filename = apache.conf.in
md5sum =
2ed3c4e9b9d58d2e57cda227bdd454d2
md5sum =
3d55f7c9c4fc7279f06bfe6313a78a4b
[template-re6st-registry-conf]
filename = re6st-registry.conf.in
md5sum = b0f0facfea82a4481f4fb5b0b263e09a
[template-wrapper]
filename = wrapper.in
md5sum = 7633bdfc0e638ff9979f963fbbca8f13
[template-registry-run]
filename = registry-run.in
md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
md5sum = feb4b3318f37414d1bf3d16a03aec93d
software/re6stnet/instance-re6stnet-input-schema.json
View file @
6f2caaa2
...
...
@@ -78,6 +78,11 @@
"description"
:
"Prevent tunnelling accross borders of listed countries"
,
"type"
:
"string"
,
"default"
:
""
},
"backup-repository"
:
{
"title"
:
"Git backup repository"
,
"description"
:
"URL of Git repository where backups are pushed. Nothing is pushed if empty."
,
"type"
:
"string"
}
}
}
software/re6stnet/instance-re6stnet.cfg.in
View file @
6f2caaa2
{% set bin_directory = parameter_dict['bin-directory'] -%}
{% set python_bin = parameter_dict['python-executable'] -%}
{% set re6st_registry = parameter_dict['re6st-registry'] -%}
{% set re6stnet = parameter_dict['re6stnet'] -%}
{% set publish_dict = {} -%}
{% set part_list = [] -%}
{% set ipv6 = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set uri_scheme = slapparameter_dict.get('uri-scheme', 'http') -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
[directory]
recipe = slapos.cookbook:mkdirectory
...
...
@@ -19,35 +14,16 @@ log = ${:var}/log
services = ${:etc}/service
script = ${:etc}/run
run = ${:var}/run
ca-dir = ${:etc}/ssl
requests = ${:ca-dir}/requests
private = ${:ca-dir}/private
certs = ${:ca-dir}/certs
newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
re6st = ${:srv}/res6stnet
[re6stnet-dirs]
recipe = slapos.cookbook:mkdirectory
registry = ${directory:re6st}/registry
data = ${directory:srv}/re6stnet
log = ${directory:log}/re6stnet
conf = ${directory:etc}/re6stnet
ssl = ${:conf}/ssl
token = ${:conf}/token
run = ${directory:run}/re6stnet
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_bin }}/openssl
ca-dir = ${directory:ca-dir}
requests-directory = ${directory:requests}
wrapper = ${directory:services}/certificate_authority
ca-private = ${directory:private}
ca-certs = ${directory:certs}
ca-newcerts = ${directory:newcerts}
ca-crl = ${directory:crl}
[apache-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-apache-conf'] }}
...
...
@@ -64,56 +40,33 @@ context =
key access_log :access-log
key error_log :error-log
key pid_file :pid-file
raw certificate ${directory:certs}/apache.crt
raw key ${directory:private}/apache.key
raw ipv6 {{ ipv6 }}
raw uri_scheme {{ uri_scheme }}
{% set apache_wrapper = '${directory:services}/httpd' -%}
{% if uri_scheme == 'https' -%}
{% set apache_wrapper = '${directory:bin}/httpd_raw' -%}
{% endif -%}
[apache-httpd]
recipe = slapos.cookbook:wrapper
wrapper-path = {{ apache_wrapper }}
command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
{% if uri_scheme == 'https' %}
[apache-ca]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
executable = ${apache-httpd:wrapper-path}
wrapper = ${directory:bin}/httpd
key-file = ${certificate-authority:ca-private}/apache.key
cert-file = ${certificate-authority:ca-certs}/apache.crt
[{{ section('apache-ca-service') }}]
recipe = slapos.cookbook:wrapper
command-line = ${apache-ca:wrapper}
wrapper-path = ${directory:services}/httpd
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% endif %}
command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
[apache-httpd-graceful]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-wrapper'] }}
rendered = ${directory:script}/httpd-graceful
mode = 0700
context =
raw content {{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}; if [ $? -eq 0 ]; then kill -USR1 $(cat ${apache-conf:pid-file}); fi
raw dash {{ dash_binary }}
template = inline:{{'{{content}}'}}
context = key content :script
script =
#!/bin/sh -e
{{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}
{{ bin_directory }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
[logrotate-apache]
< = logrotate-entry-base
name = apache
log = ${apache-conf:error-log} ${apache-conf:access-log}
post = test ! -s ${apache-conf:pid-file} || {{
parameter_dict['bin-directory']
}}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
post = test ! -s ${apache-conf:pid-file} || {{
bin_directory
}}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
[re6st-registry-conf-dict]
port = 9201
ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }}
db = ${re6stnet-dirs:registry}/registry.db
db = ${re6stnet-dirs:data}/registry.db
ca = ${re6stnet-dirs:ssl}/re6stnet.crt
key = ${re6stnet-dirs:ssl}/re6stnet.key
dh = ${re6stnet-dirs:ssl}/dh.pem
...
...
@@ -135,18 +88,35 @@ same-country = {{ slapparameter_dict.get('same-country', '') }}
[re6st-registry-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-re6st-registry-conf'] }}
rendered = ${
directory:etc}/re6st-
registry.conf
rendered = ${
re6stnet-dirs:conf}/
registry.conf
context = section parameter_dict re6st-registry-conf-dict
depends = ${re6st-compat:recipe}
[re6st-compat]
recipe = slapos.recipe.build
update =
import errno, os
res6stnet = self.buildout['directory']['srv'] + '/res6stnet'
try:
os.rename(res6stnet + '/registry', self.buildout['re6stnet-dirs']['data'])
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
os.rmdir(res6stnet)
[re6st-registry-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-registry-run'] }}
rendered = ${directory:services}/re6st-registry
pid-file = ${directory:run}/registry.pid
context =
key pid_file :pid-file
raw re6st_command {{ re6st_registry }}
key re6st_conf re6st-registry-conf:rendered
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/re6st-registry
pidfile = ${directory:run}/registry.pid
command-line = {{ bin_directory }}/re6st-registry @${re6st-registry-conf:rendered}
[cron-entry-re6st-backup]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-backup
time = hourly
command = {{ parameter_dict['re6stnet-backup'] }} ${logrotate-directory:backup}/re6stnet ${re6st-registry-conf-dict:db} {{ slapparameter_dict.get('backup-repository', '') }}
[re6st-registry]
recipe = slapos.cookbook:re6stnet.registry
...
...
@@ -167,7 +137,7 @@ key-file = ${re6st-registry-conf-dict:key}
cert-file = ${re6st-registry-conf-dict:ca}
dh-file = ${re6st-registry-conf-dict:dh}
slave-instance-list =
${slap-parameter:slave_instance_list
}
slave-instance-list =
{{ slapparameter_dict.get('slave_instance_list', '{}') }
}
environment =
PATH={{ openssl_bin }}
...
...
@@ -181,14 +151,14 @@ command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-check-token
frequency = */5 * * * *
time = *:0/5
command = {{ python_bin }} ${re6st-registry:manager-wrapper}
[logrotate-entry-re6stnet]
< = logrotate-entry-base
name = re6stnet
log = ${re6st-registry-conf-dict:logfile}
post =
test ! -s ${re6st-registry-wrapper:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${re6st-registry-wrapper:pid-
file} -s USR1
post =
[ ! -s ${re6st-registry-wrapper:pidfile} ] || {{ bin_directory }}/slapos-kill --pidfile ${re6st-registry-wrapper:pid
file} -s USR1
[port-redirection]
recipe = slapos.recipe.template:jinja2
...
...
@@ -214,7 +184,7 @@ name = apache-re6st-registry.py
config-hostname = ${apache-conf:ipv6}
config-port = ${apache-conf:port}
{% do publish_dict.__setitem__('re6stry-url',
uri_scheme ~ '
://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
{% do publish_dict.__setitem__('re6stry-url',
'http
://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
{% do publish_dict.__setitem__('re6stry-local-url', 'http://${re6st-registry:ipv4}:${re6st-registry:port}/') -%}
{% do publish_dict.__setitem__('slave-amount', '${re6st-registry:slave-amount}') -%}
[publish]
...
...
@@ -230,12 +200,12 @@ extends =
{{ logrotate_cfg }}
parts =
certificate-authority
logrotate-apache
logrotate-entry-re6stnet
re6stnet-manage
cron-entry-logrotate
cron-entry-re6st-manage
cron-entry-re6st-backup
apache-httpd
apache-httpd-graceful
publish
...
...
@@ -244,12 +214,3 @@ parts =
re6st-registry-promise
apache-registry-promise
monitor-base
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameter]
slave_instance_list = {}
software/re6stnet/instance.cfg.in
View file @
6f2caaa2
...
...
@@ -16,27 +16,21 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extra-context =
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key ipv6_set slap-configuration:ipv6
key ipv4_set slap-configuration:ipv4
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
raw logrotate_cfg {{ template_logrotate_base }}
raw dash_binary {{ dash_location }}/bin/dash
raw openssl_bin {{ openssl_location}}/bin
${:extra-context}
[dynamic-template-re6stnet-parameters]
bin-directory = {{ bin_directory }}
python-executable = {{ python_with_eggs }}
re6st-registry = {{ bin_directory }}/re6st-registry
re6stnet = {{ bin_directory }}/re6stnet
template-apache-conf = {{ template_apache_conf }}
template-wrapper = {{ template_wrapper }}
apache-location = {{ apache_location }}
re6stnet-backup = {{ re6stnet_backup }}
template-re6st-registry-conf = {{ template_re6st_registry_conf }}
template-registry-run = {{ template_registry_run }}
[dynamic-template-re6stnet]
< = jinja2-template-base
...
...
@@ -48,7 +42,9 @@ extra-context =
raw monitor2_template_rendered {{ monitor2_template_rendered }}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
recipe = slapos.cookbook:s
witch-s
oftwaretype
default =
${dynamic-template-re6stnet:rendered}
default =
dynamic-template-re6stnet:rendered
registry = ${:default}
# BBB
RootSoftwareInstance = ${:default}
software/re6stnet/re6st-registry.conf.in
View file @
6f2caaa2
port {{ parameter_dict['port'] }}
4 {{ parameter_dict['ipv4'] }}
authorized-origin {{ parameter_dict['ipv4'] }}
# 6 {{ parameter_dict['ipv6'] }}
6 ''
db {{ parameter_dict['db'] }}
ca {{ parameter_dict['ca'] }}
key {{ parameter_dict['key'] }}
...
...
software/re6stnet/registry-run.in
deleted
100644 → 0
View file @
b4823d48
#!/bin/bash
echo
$$
>
{{
pid_file
}}
exec
{{
re6st_command
}}
@
{{
re6st_conf
}}
software/re6stnet/software.cfg
View file @
6f2caaa2
...
...
@@ -2,13 +2,10 @@
extends =
buildout.hash.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg
../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/apache/buildout.cfg
../../component/
pycurl
/buildout.cfg
../../component/
git
/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg
...
...
@@ -17,10 +14,22 @@ extends =
parts +=
slapos-cookbook
dash
template
[re6st-eggs]
[re6stnet-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/re6stnet.git
git-executable = ${git:location}/bin/git
[re6stnet-setup-env]
PATH = ${git:location}/bin:%(PATH)s
[re6stnet-develop]
recipe = zc.recipe.egg:develop
setup = ${re6stnet-repository:location}
environment = re6stnet-setup-env
[re6stnet]
recipe = zc.recipe.egg
interpreter = python-${:_buildout_section_name_}
eggs =
...
...
@@ -28,6 +37,33 @@ eggs =
${python-cryptography:egg}
pyOpenSSL
re6stnet
initialization =
import os
path = os.environ.get('PATH', '')
os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
depends = ${re6stnet-develop:recipe}
[re6stnet-backup]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/re6st-backup
template =
inline:#!/bin/sh -e
PATH={{git_location}}/bin:{{sqlite3_location}}/bin:$PATH
cd "$1" || {
rm -rf "$1.new"
git init --bare "$1.new"
rm "$1.new"/description "$1.new"/hooks/*
mv "$1.new" "$1"
cd "$1"
}
h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
git update-index --add --cacheinfo 0644 "$h" registry.sql
git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
-c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
[ ! "$3" ] || git push --mirror "$3"
context =
key git_location git:location
key sqlite3_location sqlite3:location
[download-base]
recipe = slapos.recipe.build:download
...
...
@@ -43,17 +79,15 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key apache_location apache:location
key dash_location dash:location
key logrotate_location logrotate:location
key openssl_location openssl:location
key re6stnet_backup re6stnet-backup:rendered
key template_apache_conf template-apache-conf:target
key template_re6stnet template-re6stnet:target
key template_re6st_registry_conf template-re6st-registry-conf:target
key template_logrotate_base template-logrotate-base:rendered
key template_wrapper template-wrapper:target
key template_registry_run template-registry-run:target
key monitor2_template_rendered monitor2-template:rendered
raw python_with_eggs ${buildout:bin-directory}/${re6st
-eggs
:interpreter}
raw python_with_eggs ${buildout:bin-directory}/${re6st
net
:interpreter}
[template-re6stnet]
< = download-base
...
...
@@ -64,15 +98,5 @@ context =
[template-re6st-registry-conf]
< = download-base
[template-wrapper]
< = download-base
[template-registry-run]
< = download-base
[versions]
re6stnet = 0.551
# Required by:
# re6stnet==0.533
miniupnpc = 1.9
software/re6stnet/test/test.py
View file @
6f2caaa2
...
...
@@ -29,7 +29,6 @@ import os
import
requests
import
json
from
slapos.recipe.librecipe
import
generateHashFromFiles
from
slapos.testing.testcase
import
makeModuleSetUpAndTestCaseClass
setUpModule
,
Re6stnetTestCase
=
makeModuleSetUpAndTestCaseClass
(
...
...
@@ -57,35 +56,3 @@ class TestPortRedirection(Re6stnetTestCase):
'srcPort'
:
9201
,
'destPort'
:
9201
,
},
portredir_config
[
0
])
class
ServicesTestCase
(
Re6stnetTestCase
):
@
classmethod
def
getInstanceParameterDict
(
cls
):
return
{
'uri-scheme'
:
'https'
}
def
test_hashes
(
self
):
hash_files
=
[
'software_release/buildout.cfg'
,
]
expected_process_names
=
[
'httpd-{hash}-on-watch'
,
]
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
process_names
=
[
process
[
'name'
]
for
process
in
supervisor
.
getAllProcessInfo
()
]
hash_files
=
[
os
.
path
.
join
(
self
.
computer_partition_root_path
,
path
)
for
path
in
hash_files
]
for
name
in
expected_process_names
:
h
=
generateHashFromFiles
(
hash_files
)
expected_process_name
=
name
.
format
(
hash
=
h
)
self
.
assertIn
(
expected_process_name
,
process_names
)
software/re6stnet/wrapper.in
deleted
100644 → 0
View file @
b4823d48
#!{{ dash }}
{{ content }}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment