Commit 11e0ba60 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 2250b89f 886d9134
......@@ -37,18 +37,14 @@ pyyaml = ${pyyaml-download:target}
pyaml = ${pyaml-download:target}
install =
import os
call([options['python'], '-m', 'venv', '--clear', location])
import os, sys
call([sys.executable, '-m', 'venv', '--clear', location])
pip = os.path.join(location, 'bin', 'pip')
call([pip, 'install', '--no-index', options['pyyaml'], options['pyaml']])
call([pip, 'uninstall', '-y', 'pip', 'setuptools'])
# selftest
python = os.path.join(location, 'bin', 'python3')
python = os.path.join(location, 'bin', 'python')
call([python, '-c', 'import yaml'])
python = ${python3:executable}
[librbd-python:python3]
python = ${buildout:executable}
[librbd]
# CEPH only for librbd
......
......@@ -67,7 +67,7 @@ environment =
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig${:PKG_CONFIG_PATH-rbd}
[qemu:[int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]]
[qemu:python2 or [int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]]
configure-rbd =
CFLAGS-rbd =
LDFLAGS-rbd =
......
......@@ -38,18 +38,14 @@ markupsafe = ${systemd-markupsafe-download:target}
jinja2 = ${systemd-jinja2-download:target}
install =
import os
call([options['python'], '-m', 'venv', '--clear', location])
import os, sys
call([sys.executable, '-m', 'venv', '--clear', location])
pip = os.path.join(location, 'bin', 'pip')
call([pip, 'install', '--no-index', options['markupsafe'], options['jinja2']])
call([pip, 'uninstall', '-y', 'pip', 'setuptools'])
# selftest
python = os.path.join(location, 'bin', 'python3')
python = os.path.join(location, 'bin', 'python')
call([python, '-c', 'import jinja2'])
python = ${python3:executable}
[systemd-python:python3]
python = ${buildout:executable}
[systemd]
recipe = slapos.recipe.build
......
[instance-profile]
filename = instance.cfg.in
md5sum = f0714597e23ebcb0f2a1368cd065a84e
md5sum = 17004b2adb98b545b16c6be60e8165e8
......@@ -7,7 +7,6 @@
parts =
publish-connection-parameter
download-plc
compile-plc
beremiz-runtime
#beremiz-runtime-promise
......@@ -21,13 +20,6 @@ recipe = slapos.recipe.build:download-unpacked
offline = false
url = ${instance-parameter:configuration.runtime_plc_url}
[compile-plc]
recipe = plone.recipe.command
stop-on-error = true
command =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py --project-home ${directory:home}/parts/download-plc/ build
update-command = ${:command}
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
......@@ -54,8 +46,19 @@ log = ${:var}/log
[beremiz-runtime]
logfile = ${directory:log}/beremiz-runtime.log
recipe = slapos.cookbook:wrapper
# adding BEREMIZPYTHONPATH is needed so we can override the Beremiz'
# internal code which tries to use sys.executable to spawn processes
# and in the context of SlapOS it's a plain Python without needed modules
# adding LIBRARY_PATH / COMPILER_PATH is needed so that gcc can find
# needed libraries and tools inside SlapOS context
environment =
BEREMIZPYTHONPATH = {{ buildout['bin-directory'] }}/pythonwitheggs
PATH=$PATH:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin
LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/x86_64-linux-gnu:/usr/lib:/lib/x86_64-linux-gnu/:/lib:/usr/lib/x86_64-linux-gnu/
COMPILER_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/gcc/x86_64-linux-gnu/
command-line =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_service.py -a ${instance-parameter:configuration.autostart} -p ${instance-parameter:configuration.port} -i ${instance-parameter:configuration.interface} -x 1 ${directory:home}/parts/download-plc/build
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py -k --project-home ${directory:home}/parts/download-plc/ build transfer run
wrapper-path = ${directory:service}/beremiz-runtime
[beremiz-runtime-promise]
......
......@@ -17,7 +17,7 @@ environment +=
[template]
recipe = slapos.recipe.template:jinja2
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = d7506e861ef87977eaa554b8928d2c99
md5sum = 2e90443314d2036f7077898ab49bfec1
# XXX: "template.cfg" is hardcoded in instanciation recipe
output = ${buildout:directory}/template.cfg
context =
......
......@@ -182,8 +182,9 @@ And now it will be safe to use the paths in ``external-disk`` parameter::
}
}
Of course ``external-disk-number`` and ``external-disk-size`` HAVE TO be removed
from instance parameters before continuing.
Of course ``external-disk-number``, ``external-disk-size`` and
``external-disk-format`` HAVE TO be removed from instance parameters before
continuing.
For now such configuration will lead to no starting kvm process, so after
parameters are updated in SlapOS Master **and** are processed on the Compute
......
============
Edge testing
============
``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
In order to monitor an url one need to:
* request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
* request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 0d26da3f83b293efbf330fd2b0350bf6
md5sum = 072b276e0b0dd4b4a96a50348a04c7a7
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
md5sum = 8e0c4041f680312ff054687d7f28275a
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a16fb139d9f3d01a88f499450fac50b4
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 7b927af12934c228d2763c5a40c634b5
[network-bench-cfg]
filename = network_bench.cfg.in
md5sum = 4f4d1a0afc225656278154e453587aea
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"region-dict": {
"title": "Regions",
"description": "Defines regions of the cluster",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the node of the region. Can be used to destroy not needed regions.",
"type": "string",
"default": "started",
"enum": [
"started",
"stopped",
"destroyed"
]
},
"sla-computer_guid": {
"title": "GUID of the computer on which this region shall be deployed",
"description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"type": "string",
"default": ""
},
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
},
"nameserver-list": {
"default": [],
"title": "Nameservers (backward compatibility)",
"description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check (backward compatibility)",
"description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com/",
"type": "string"
},
"region-dict": {
"title": "Applicable Regions",
"description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"type": "string",
"default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
},
"failure-amount": {
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
}
}
}
{%- if slap_software_type == software_type %}
{%- set CONFIGURATION = {} %}
{%- for k, v in sorted(slap_configuration.items()) %}
{%- if k.startswith('configuration.') %}
{%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %}
{%- endfor %}
{%- set slave_instance_dict = {} %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.setdefault('check-http-header-dict', {}) %}
{%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set URL_LIST = [] %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- do URL_LIST.append(slave['url']) %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
<= monitor-promise-base
promise = check_surykatka_json
name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %}
[surykatka-bot-promise-{{ class }}]
<= monitor-promise-base
promise = check_surykatka_json
name = surykatka-bot-promise-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
output = ${directory:etc}/surykatka-{{ class }}.ini
url = {{ template_surykatka_ini }}
url_list = {{ dumps(URL_LIST) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key url_list :url_list
key timeout :timeout
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config-{{ class }}:output}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template
json = ${surykatka-config-{{ class }}:json}
inline =
#!/bin/sh
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
output = ${monitor-directory:bin}/${:_buildout_section_name_}
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:output}
{%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
monitor-base
publish-connection-information
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %} {#- if slap_software_type == software_type #}
{%- if slap_software_type == software_type %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
url = {{ template_json_edgetest_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
{%- set edgebot_software_type = 'edgebot' %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- if 'region-dict' not in slapparameter_dict %}
{#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- set number = {'i': 1} %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %}
{#- Note: monitor-httpd-port will vary on regions being added and removed,
but this is accepted, as it's only internal trick #}
{%- do request_dict.__setitem__(
request_section_title,
{
'config': {
'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name,
'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': region_parameter_dict.get('state', 'started'),
}) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %}
[replicate]
<= slap-connection
recipe = slapos.cookbook:request.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
{%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request['name'] }}
state = {{ edgebot_request['state'] }}
{%- if edgebot_request['state'] != 'destroyed' %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
return = monitor-base-url
{%- endif %}
{%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- for parameter, value in edgebot_request['sla'].items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- endfor %}
# Publish information for each slave
{%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish.serialised
available-region-list = {{ dumps(list(active_region_list)) }}
-slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
......@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:output
edgetest = instance-base-edgetest:output
edgebot = instance-base-edgebot:output
edgetest-basic = instance-edgetest-basic:output
RootSoftwareInstance = $${:default}
......@@ -54,45 +52,6 @@ extra-context =
key template_surykatka_ini surykatka:ini
key surykatka_binary surykatka:binary
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
url = ${template-monitor-edgetest:target}
output = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
section slap_configuration slap-configuration
raw software_type edgetest
key instance_base_monitor instance-base-monitor:output
key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_edgetest_test ${json-test-template:target}
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
url = ${template-monitor-edgebot:target}
output = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary surykatka:binary
key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
......
......@@ -13,7 +13,6 @@ parts =
network-bench-cfg
json-test-template
template
template-monitor-edgetest
template-monitor
[python]
......@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-monitor-edgebot]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-surykatka-ini]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
......
......@@ -17,24 +17,6 @@
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest": {
"title": "Edge Test [OBSOLETE]",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
},
"edgetest-slave": {
"title": "Edge Test Slave [OBSOLETE]",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 3
}
}
}
This diff is collapsed.
......@@ -24,7 +24,7 @@ md5sum = b10c10a9408a934052fd0960f31a75fc
[template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = b5a0f5e258fd91573425b00027ca11b4
md5sum = c5a82279b6ada358c74a90c3b660bf55
[template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg
......@@ -36,7 +36,7 @@ md5sum = 2cfe5679374559ab94478b987d646d83
[template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = 771c6b550ea558c708f207c75ff00f49
md5sum = eabbdb8ced563180f0bba8ca06e4bac4
[template-lte-mme]
_update_hash_filename_ = instance-mme.jinja2.cfg
......@@ -48,7 +48,7 @@ md5sum = dcaac06553a3222b14c0013a13f4a149
[enb.jinja2.cfg]
filename = config/enb.jinja2.cfg
md5sum = cd93c9c7bff2bcdd1cc01ad1cdf0c88c
md5sum = f5848c48f573c79615f595b07c358ce8
[sib23.asn]
filename = config/sib23.asn
......@@ -56,7 +56,7 @@ md5sum = b377dac7f1fcf94fb9ce9ebed617f36a
[gnb.jinja2.cfg]
filename = config/gnb.jinja2.cfg
md5sum = 91005da479a848798d128dbfd407d2c7
md5sum = b3fb1b32aa1b151892eeb87a03e1d152
[ltelogs.jinja2.sh]
filename = ltelogs.jinja2.sh
......@@ -64,7 +64,7 @@ md5sum = 1ba2e065bdf14a6411e95e80db17dcfd
[mme.jinja2.cfg]
filename = config/mme.jinja2.cfg
md5sum = da10ef4c61532364c758c7845614c4d6
md5sum = a96c33a0a2a301f204cac8642267ad97
[ims.jinja2.cfg]
filename = config/ims.jinja2.cfg
......
......@@ -4,7 +4,7 @@
#define N_ANTENNA_UL 2
{
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}",
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/enb.log",
rf_driver: {
......
......@@ -6,7 +6,7 @@
#define NR_LONG_PUCCH_FORMAT 2 // Values: 2, 3, 4
{
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}",
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/gnb.log",
rf_driver: {
......
{
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}",
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/mme.log",
{% if slapparameter_dict.get('external_enb_gnb', '') %}
......
......@@ -5,7 +5,6 @@ parts =
lte-enb-config
lte-enb-service
sdr-busy-promise
remove-tmp
monitor-base
publish-connection-information
......@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[remove-tmp]
# Remove old logs stored in tmp directory to prevent disk from becoming full
recipe = plone.recipe.command
command = rm -rf ${directory:tmp}/*
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
......
......@@ -5,7 +5,6 @@ parts =
lte-gnb-config
lte-enb-service
sdr-busy-promise
remove-tmp
monitor-base
publish-connection-information
......@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[remove-tmp]
# Remove old logs stored in tmp directory to prevent disk from becoming full
recipe = plone.recipe.command
command = rm -rf ${directory:tmp}/*
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
......
......@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 6fc82c4b645bceb97ba7b6f5b56d161b
md5sum = dc01433ffed82386a3feee0b34767bed
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in
md5sum = cefcb8c7d17367b14414314ffd325c26
md5sum = a995bcd2751739710ba01a269d3d5b46
......@@ -110,6 +110,9 @@ output = ${directory:bin}/update-proxysql-config
'cluster_id': instance_dict.get('cluster_id', default_slave_cluster_id),
'require_ssl': True
} -%}
{% if not slave_dict['cluster_id'] in cluster_name_list -%}
{% do slave_dict.__setitem__('cluster_id', default_slave_cluster_id) -%}
{% endif -%}
{% do slave_information_list.append(slave_dict) -%}
{% do database_slave_dict[slave_dict['cluster_id']].append(slave_dict) -%}
{% do db_name_dict[slave_dict['cluster_id']].append([slave_dict['name'], slave_dict['user']]) -%}
......
......@@ -2,27 +2,17 @@
set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
revoke_user () {
DB=$1
UNAME=$2
if [ ! -z "$UNAME" ]; then
echo "Revoking all grants for user '$USER'";
run_mysql -Be "
set sql_log_bin=0;
REVOKE ALL PRIVILEGES, GRANT OPTION FROM '$UNAME';
DROP USER IF EXISTS '$UNAME'@'%';
DROP USER IF EXISTS '$UNAME'@'localhost';
......@@ -32,10 +22,10 @@ revoke_user () {
fi
}
# Only write or delete on master database else, we break replication.
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
# Disable SQL_LOG_BIN, repman_slave_definition database doesn't need to be replicated
cat << EOF > {{ tmp_dir }}/.script.sql
cat << EOF > {{ tmp_dir }}/.script.sql
SET @@SESSION.SQL_LOG_BIN=0;
use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` (
......@@ -54,34 +44,33 @@ REPLACE INTO \`slave\` VALUES ('{{ name }}', true, '{{ user }}');
EOF
# Update requested slaves database
OUTPUT="{{ tmp_dir }}/removed_db.txt"
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
run_mysql -NBe "SELECT name, user FROM repman_slave_definition.slave WHERE state=false" > $OUTPUT
# Update requested slaves database
OUTPUT="{{ tmp_dir }}/removed_db.txt"
run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql
run_mysql -NBe "set sql_log_bin=0;SELECT name, user FROM repman_slave_definition.slave WHERE state=false" > $OUTPUT
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
DBNAME=$(run_mysql --skip-column-names -Be "set sql_log_bin=0;SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$?
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME"
exit $RET
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
if [ -z "$DBNAME" ]; then
echo "No database for slave to remove.";
fi
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
USER=$(grep -oP "$NAME\s*\K\w+" $OUTPUT);
if [ ! -z "$USER" ]; then
revoke_user $NAME $USER;
fi
echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then
USER=$(grep -oP "$NAME\s*\K\w+" $OUTPUT);
if [ ! -z "$USER" ]; then
revoke_user $NAME $USER;
fi
done
fi
echo "Deleting database $NAME..."
run_mysql -e "set sql_log_bin=0;DROP DATABASE IF EXISTS $NAME";
run_mysql -e "set sql_log_bin=0;DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done."
fi
done
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment