Skip to content

slapos
slapos
Commit ba7699ff authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

software/monitor: Drop obsoleted edgetest software type 

edgebot is dropped too, as it was shared instance implementation for edgetest.
parent b69af7c3
Pipeline #22893 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 1 addition and 1745 deletions
    software/monitor/README.edgetest.rst deleted 100644 → 0
    View file @ b69af7c3
    ============
    Edge testing
    ============
    ``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
    It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
    ``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
    ``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
    In order to monitor an url one need to:
    * request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
    * request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
    software/monitor/buildout.hash.cfg
    View file @ ba7699ff
    ...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
    # not need these here). # not need these here).
    [template] [template]
    filename = instance.cfg filename = instance.cfg
    md5sum = 0d26da3f83b293efbf330fd2b0350bf6 md5sum = 072b276e0b0dd4b4a96a50348a04c7a7
    [template-monitor] [template-monitor]
    _update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
    ...@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 ...@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
    _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
    md5sum = 8e0c4041f680312ff054687d7f28275a md5sum = 8e0c4041f680312ff054687d7f28275a
    [template-monitor-edgetest]
    _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
    md5sum = a16fb139d9f3d01a88f499450fac50b4
    [template-monitor-edgebot]
    _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
    md5sum = 7b927af12934c228d2763c5a40c634b5
    [network-bench-cfg] [network-bench-cfg]
    filename = network_bench.cfg.in filename = network_bench.cfg.in
    md5sum = 4f4d1a0afc225656278154e453587aea md5sum = 4f4d1a0afc225656278154e453587aea
    ......
    software/monitor/instance-edgetest-input-schema.json deleted 100644 → 0
    View file @ b69af7c3
    {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "properties": {
    "region-dict": {
    "title": "Regions",
    "description": "Defines regions of the cluster",
    "patternProperties": {
    ".*": {
    "properties": {
    "state": {
    "title": "State",
    "description": "State of the node of the region. Can be used to destroy not needed regions.",
    "type": "string",
    "default": "started",
    "enum": [
    "started",
    "stopped",
    "destroyed"
    ]
    },
    "sla-computer_guid": {
    "title": "GUID of the computer on which this region shall be deployed",
    "description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
    "type": "string",
    "default": ""
    },
    "nameserver-list": {
    "default": [],
    "title": "Nameservers",
    "description": "List of nameservers to use.",
    "type": "array"
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Default Frontend IPs to check",
    "description": "List of default frontend IPs to check, if empty no constraint is used.",
    "type": "array"
    }
    },
    "type": "object"
    }
    },
    "type": "object"
    },
    "nameserver-list": {
    "default": [],
    "title": "Nameservers (backward compatibility)",
    "description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
    "type": "array"
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Default Frontend IPs to check (backward compatibility)",
    "description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
    "type": "array"
    }
    }
    }
    software/monitor/instance-edgetest-slave-input-schema.json deleted 100644 → 0
    View file @ b69af7c3
    {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "properties": {
    "url": {
    "title": "URL to check",
    "description": "URL to check, like https://example.com/",
    "type": "string"
    },
    "region-dict": {
    "title": "Applicable Regions",
    "description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
    "patternProperties": {
    ".*": {
    "properties": {
    "state": {
    "title": "State",
    "description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
    "type": "string",
    "default": "present",
    "enum": [
    "present"
    ]
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Frontend IPs to check",
    "description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
    "type": "array"
    }
    },
    "type": "object"
    }
    },
    "type": "object",
    "default": {}
    },
    "check-status-code": {
    "title": "HTTP Code Check",
    "description": "Expected response HTTP Code.",
    "type": "number",
    "default": 200,
    "minimum": 100,
    "maximum": 599
    },
    "check-certificate-expiration-days": {
    "title": "Certificate Expiration Check (days)",
    "description": "Amount of days to consider certificate as being to-be-expired.",
    "type": "number",
    "default": 15,
    "minimum": 1
    },
    "check-maximum-elapsed-time": {
    "title": "Maximum Elapsed Check (seconds)",
    "description": "Maximum elapsed time for a site to reply to be considered good.",
    "type": "number",
    "default": 2,
    "minimum": 1
    },
    "check-http-header-dict": {
    "title": "HTTP Header Check",
    "description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
    "type": "object",
    "default": {}
    },
    "failure-amount": {
    "title": "Failure Amount",
    "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
    "type": "number",
    "default": 2,
    "minimum": 1
    },
    "check-frontend-ip-list": {
    "title": "Frontend IPs to check (backward compatibility)",
    "description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
    "type": "array"
    }
    }
    }
    software/monitor/instance-monitor-edgebot.cfg.jinja2 deleted 100644 → 0
    View file @ b69af7c3
    {%- if slap_software_type == software_type %}
    {%- set CONFIGURATION = {} %}
    {%- for k, v in sorted(slap_configuration.items()) %}
    {%- if k.startswith('configuration.') %}
    {%- do CONFIGURATION.__setitem__(k[14:], v) %}
    {%- endif %}
    {%- endfor %}
    {%- set slave_instance_dict = {} %}
    {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
    {%- if extra_slave_instance_list %}
    {#- Create slaves to process with setting up defaults #}
    {%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
    {%- do slave.setdefault('check-status-code', 200) %}
    {%- do slave.setdefault('check-http-header-dict', {}) %}
    {%- do slave.setdefault('check-certificate-expiration-days', 15) %}
    {%- do slave.setdefault('failure-amount', 2) %}
    {%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
    {%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
    {%- if 'url' in slave %}
    {%- set class = slave['check-maximum-elapsed-time'] %}
    {%- if class not in slave_instance_dict %}
    {%- do slave_instance_dict.__setitem__(class, []) %}
    {%- endif %}
    {%- do slave_instance_dict[class].append(slave) %}
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- set part_list = [] %}
    {%- for class, slave_instance_list in slave_instance_dict.items() %}
    {#- class is used to separate surykatka with different timeouts #}
    {%- set URL_LIST = [] %}
    {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
    {%- do URL_LIST.append(slave['url']) %}
    {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
    {%- do part_list.append(part_id) %}
    {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
    [{{part_id}}]
    <= monitor-promise-base
    promise = check_surykatka_json
    name = {{ safe_name }}.py
    config-report = http_query
    config-url = {{ slave['url'] }}
    config-status-code = {{ slave['check-status-code'] }}
    config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
    config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
    config-failure-amount = {{ slave['failure-amount'] }}
    config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
    config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
    config-json-file = ${surykatka-config-{{ class }}:json}
    {%- endfor %}
    [surykatka-bot-promise-{{ class }}]
    <= monitor-promise-base
    promise = check_surykatka_json
    name = surykatka-bot-promise-{{ class }}.py
    config-report = bot_status
    config-json-file = ${surykatka-config-{{ class }}:json}
    [surykatka-config-{{ class }}]
    recipe = slapos.recipe.template:jinja2
    db = ${directory:srv}/surykatka-{{ class }}.db
    output = ${directory:etc}/surykatka-{{ class }}.ini
    url = {{ template_surykatka_ini }}
    url_list = {{ dumps(URL_LIST) }}
    nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
    json = ${directory:srv}/surykatka-{{ class }}.json
    {#- timeout is just a bit bigger than class time #}
    timeout = {{ int(class) + 2 }}
    context =
    import json_module json
    key db :db
    key nameserver_list :nameserver_list
    key url_list :url_list
    key timeout :timeout
    [surykatka-{{ class }}]
    recipe = slapos.cookbook:wrapper
    config = ${surykatka-config-{{ class }}:output}
    command-line =
    {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
    wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
    hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
    [surykatka-status-json-{{ class }}]
    recipe = slapos.recipe.template
    json = ${surykatka-config-{{ class }}:json}
    inline =
    #!/bin/sh
    {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
    output = ${monitor-directory:bin}/${:_buildout_section_name_}
    [cron-entry-surykatka-status-{{ class }}]
    recipe = slapos.cookbook:cron.d
    cron-entries = ${directory:etc}/cron.d
    name = surykatka-status-{{ class }}
    frequency = */2 * * * *
    command = ${surykatka-status-json-{{ class }}:output}
    {%- do part_list.append('surykatka-%i'% (class,)) %}
    {%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
    {%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
    {%- endfor %}
    [buildout]
    extends = {{ monitor_template_output }}
    parts =
    cron
    monitor-base
    publish-connection-information
    {% for part_id in sorted(part_list) %}
    {{ part_id }}
    {% endfor %}
    eggs-directory = {{ eggs_directory }}
    develop-eggs-directory = {{ develop_eggs_directory }}
    offline = true
    [publish-connection-information]
    recipe = slapos.cookbook:publish.serialised
    monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
    monitor-url = ${monitor-publish-parameters:monitor-url}
    monitor-user = ${monitor-publish-parameters:monitor-user}
    monitor-password = ${monitor-publish-parameters:monitor-password}
    [monitor-instance-parameter]
    monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
    cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
    {% if slapparameter_dict.get('monitor-username', '') -%}
    username = {{ slapparameter_dict['monitor-username'] }}
    {% endif -%}
    {% if slapparameter_dict.get('monitor-password', '') -%}
    password = {{ slapparameter_dict['monitor-password'] }}
    {% endif -%}
    interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
    [monitor-directory]
    service = ${buildout:directory}/etc/service
    var = ${buildout:directory}/var
    srv = ${buildout:directory}/srv
    server-log = ${:private}/server-log
    monitor-log = ${:private}/monitor-log
    [slap-configuration]
    recipe = slapos.cookbook:slapconfiguration.serialised
    computer = ${slap-connection:computer-id}
    partition = ${slap-connection:partition-id}
    url = ${slap-connection:server-url}
    key = ${slap-connection:key-file}
    cert = ${slap-connection:cert-file}
    [slap-parameter]
    {%- endif %} {#- if slap_software_type == software_type #}
    software/monitor/instance-monitor-edgetest.cfg.jinja2 deleted 100644 → 0
    View file @ b69af7c3
    {%- if slap_software_type == software_type %}
    [jinja2-template-base]
    recipe = slapos.recipe.template:jinja2
    output = ${buildout:directory}/${:filename}
    extra-context =
    context =
    import json_module json
    ${:extra-context}
    [slave-test-configuration]
    <=jinja2-template-base
    url = {{ template_json_edgetest_test }}
    filename = srv/monitor/private/test.json
    extensions = jinja2.ext.do
    extra-context =
    section slave_information slap-configuration
    {% set part_list = [] -%}
    {%- set edgebot_software_type = 'edgebot' %}
    {%- set slave_list_name = 'extra_slave_instance_list' %}
    {%- set request_dict = {} %}
    {%- set namebase = "edgebot" %}
    {%- if 'region-dict' not in slapparameter_dict %}
    {#- Be nice and allow to work with default configuration #}
    {%- do slapparameter_dict.__setitem__('region-dict', {
    '1': {
    'sla-computer_guid': slap_configuration['computer'],
    'state': slap_configuration['instance-state'],
    'nameserver-list': slapparameter_dict.get('nameserver-list', []),
    'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
    }
    }) %}
    {%- endif %}
    {%- set active_region_list = [] %}
    {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
    {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
    {%- if region_parameter_dict.get('state', 'started') == 'started' %}
    {%- do active_region_list.append(region_name) %}
    {%- endif %}
    {%- endfor %}
    {%- set authorized_slave_dict = {} %}
    {%- set publish_slave_dict_dict = {} %}
    {%- for slave in slave_instance_list | sort(attribute='slave_title') %}
    {%- set slave_reference = slave.pop('slave_reference') %}
    {%- set publish_dict = {'assigned-region-dict': {}} %}
    {%- if '_' in slave %}
    {%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
    {%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
    {%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
    {%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
    {%- if slave_region_dict == {} %}
    {%- for region in active_region_list %}
    {%- do slave_region_dict.__setitem__(region, {}) %}
    {%- endfor %}
    {%- endif %}
    {%- for region in slave_region_dict %}
    {%- if region in active_region_list %}
    {%- set region_info = {
    'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
    'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
    } %}
    {%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
    {%- set slave_dict = base_slave_dict.copy() %}
    {%- do slave_dict.update(region_info) %}
    {%- if region not in authorized_slave_dict %}
    {%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
    {%- else %}
    {%- do authorized_slave_dict[region].append(slave_dict) %}
    {%- endif %}
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
    {%- endfor %}
    {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
    {%- set number = {'i': 1} %}
    {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
    {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
    {%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
    {%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
    {%- do part_list.append(request_section_title) %}
    {#- Note: monitor-httpd-port will vary on regions being added and removed,
    but this is accepted, as it's only internal trick #}
    {%- do request_dict.__setitem__(
    request_section_title,
    {
    'config': {
    'monitor-httpd-port': monitor_base_port + number['i'],
    'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
    'nameserver-list': region_parameter_dict.get('nameserver-list', []),
    'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
    },
    'name': edgebot_name,
    'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
    'state': region_parameter_dict.get('state', 'started'),
    }) %}
    {%- do number.__setitem__('i', number['i'] + 1) %}
    {%- endfor %}
    [replicate]
    <= slap-connection
    recipe = slapos.cookbook:request.serialised
    config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
    config-monitor-username = ${monitor-instance-parameter:username}
    config-monitor-password = ${monitor-htpasswd:passwd}
    software-url = ${slap-connection:software-release-url}
    software-type = {{edgebot_software_type}}
    {%- set monitor_base_url_dict = {} -%}
    {% for section, edgebot_request in request_dict.items() %}
    [{{section}}]
    <= replicate
    name = {{ edgebot_request['name'] }}
    state = {{ edgebot_request['state'] }}
    {%- if edgebot_request['state'] != 'destroyed' %}
    {%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
    return = monitor-base-url
    {%- endif %}
    {%- set edgebot_configuration_dict = edgebot_request['config'] %}
    {%- for config_key, config_value in edgebot_configuration_dict.items() %}
    config-{{ config_key }} = {{ dumps(config_value) }}
    {% endfor -%}
    {%- for parameter, value in edgebot_request['sla'].items() %}
    sla-{{ parameter }} = {{ value }}
    {%- endfor %}
    {%- endfor %}
    # Publish information for each slave
    {%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
    {%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
    {%- do part_list.append(publish_section_title) %}
    [{{ publish_section_title }}]
    recipe = slapos.cookbook:publish.serialised
    available-region-list = {{ dumps(list(active_region_list)) }}
    -slave-reference = {{ slave_reference }}
    {%- for key, value in publish_dict.items() %}
    {{ key }} = {{ dumps(value) }}
    {%- endfor %}
    {% endfor %}
    [monitor-conf-parameters]
    monitor-title = Monitor
    password = ${monitor-htpasswd:passwd}
    [monitor-base-url-dict]
    {% for key, value in monitor_base_url_dict.items() -%}
    {{ key }} = {{ value }}
    {% endfor %}
    [buildout]
    extends = {{ instance_base_monitor }}
    parts +=
    slave-test-configuration
    {% for part in part_list %}
    {{ ' %s' % part }}
    {%- endfor %}
    [publish-connection-information]
    recipe = slapos.cookbook:publish.serialised
    active-region-list = {{ dumps(list(active_region_list)) }}
    sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
    sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
    {%- endif %} {#- if slap_software_type == software_type #}
    software/monitor/instance.cfg
    View file @ ba7699ff
    ...@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} ...@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
    [switch_softwaretype] [switch_softwaretype]
    recipe = slapos.cookbook:switch-softwaretype recipe = slapos.cookbook:switch-softwaretype
    default = instance-base-monitor:output default = instance-base-monitor:output
    edgetest = instance-base-edgetest:output
    edgebot = instance-base-edgebot:output
    edgetest-basic = instance-edgetest-basic:output edgetest-basic = instance-edgetest-basic:output
    RootSoftwareInstance = $${:default} RootSoftwareInstance = $${:default}
    ...@@ -54,45 +52,6 @@ extra-context = ...@@ -54,45 +52,6 @@ extra-context =
    key template_surykatka_ini surykatka:ini key template_surykatka_ini surykatka:ini
    key surykatka_binary surykatka:binary key surykatka_binary surykatka:binary
    [instance-base-edgetest]
    recipe = slapos.recipe.template:jinja2
    url = ${template-monitor-edgetest:target}
    output = $${buildout:directory}/template-monitor-base-edgetest.cfg
    extensions = jinja2.ext.do
    context = import json_module json
    import hashlib_module hashlib
    key develop_eggs_directory buildout:develop-eggs-directory
    key eggs_directory buildout:eggs-directory
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    section slap_configuration slap-configuration
    raw software_type edgetest
    key instance_base_monitor instance-base-monitor:output
    key slave_instance_list slap-configuration:slave-instance-list
    raw buildout_bin ${buildout:bin-directory}
    raw template_json_edgetest_test ${json-test-template:target}
    [instance-base-edgebot]
    recipe = slapos.recipe.template:jinja2
    url = ${template-monitor-edgebot:target}
    output = $${buildout:directory}/template-monitor-edgebot.cfg
    extensions = jinja2.ext.do
    context = import json_module json
    import hashlib_module hashlib
    key develop_eggs_directory buildout:develop-eggs-directory
    key eggs_directory buildout:eggs-directory
    section slap_configuration slap-configuration
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    raw software_type edgebot
    key surykatka_binary surykatka:binary
    key template_surykatka_ini surykatka:ini
    raw buildout_bin ${buildout:bin-directory}
    raw monitor_template_output ${monitor-template:output}
    raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
    [slap-configuration] [slap-configuration]
    recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
    computer = $${slap-connection:computer-id} computer = $${slap-connection:computer-id}
    ......
    software/monitor/software.cfg
    View file @ ba7699ff
    ...@@ -13,7 +13,6 @@ parts = ...@@ -13,7 +13,6 @@ parts =
    network-bench-cfg network-bench-cfg
    json-test-template json-test-template
    template template
    template-monitor-edgetest
    template-monitor template-monitor
    [python] [python]
    ...@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_} ...@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
    recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-monitor-edgetest]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-monitor-edgebot]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-surykatka-ini] [template-surykatka-ini]
    recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
    ......
    software/monitor/software.cfg.json
    View file @ ba7699ff
    ...@@ -17,24 +17,6 @@ ...@@ -17,24 +17,6 @@
    "response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml", "serialisation": "json-in-xml",
    "index": 1 "index": 1
    },
    "edgetest": {
    "title": "Edge Test [OBSOLETE]",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 2
    },
    "edgetest-slave": {
    "title": "Edge Test Slave [OBSOLETE]",
    "shared": true,
    "software-type": "edgetest",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-slave-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 3
    } }
    } }
    } }
    software/monitor/test/test.py
    View file @ ba7699ff
    ...@@ -32,7 +32,6 @@ import os ...@@ -32,7 +32,6 @@ import os
    import re import re
    import requests import requests
    import subprocess import subprocess
    import unittest
    import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
    from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
    from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
    ...@@ -217,1204 +216,6 @@ class EdgeMixin(object): ...@@ -217,1204 +216,6 @@ class EdgeMixin(object):
    self.assertIn('bot_status', status_json) self.assertIn('bot_status', status_json)
    class EdgeSlaveMixin(EdgeMixin, MonitorTestMixin):
    @classmethod
    def setUpClass(cls):
    # XXX we run these tests with --all as a workaround for the fact that after
    # requesting new shared instances we don't have promise to wait for the
    # processing of these shared instances to be completed.
    # The sequence is something like this:
    # - `requestEdgetestSlaves` will request edgetest partition
    # - first `waitForInstance` will process the edgetest partition, which
    # will request a edgebot partition, but without promise to wait for the
    # processing to be finished, so the first run of `slapos node instance`
    # exits with success code and `waitForInstance` return.
    # - second `waitForInstance` process the edgebot partition.
    # Once we implement a promise (or something similar) here, we should not
    # have to use --all
    cls.slap._force_slapos_node_instance_all = True
    return super().setUpClass()
    def assertConnectionParameterDict(self):
    serialised = self.requestDefaultInstance().getConnectionParameterDict()
    connection_parameter_dict = json.loads(serialised['_'])
    # tested elsewhere
    connection_parameter_dict.pop('monitor-setup-url', None)
    # comes from instance-monitor.cfg.jinja2, not needed here
    connection_parameter_dict.pop('server_log_url', None)
    self.assertEqual(
    self.expected_connection_parameter_dict,
    connection_parameter_dict
    )
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'edgetest'
    def updateSurykatkaDict(self):
    for instance_reference in self.surykatka_dict:
    for class_ in self.surykatka_dict[instance_reference]:
    update_dict = {}
    update_dict['ini-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'surykatka-%s.ini' % (class_,))
    update_dict['json-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.json' % (class_,))
    update_dict['status-json'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'bin',
    'surykatka-status-json-%s' % (class_,))
    update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
    update_dict['status-cron'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'cron.d', 'surykatka-status-%s' % (class_,))
    update_dict['db_file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.db' % (class_,))
    self.surykatka_dict[instance_reference][class_].update(update_dict)
    def assertHttpQueryPromiseContent(self, instance_reference, name, content):
    hashed = 'http-query-%s-promise.py' % (
    hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
    self.assertPromiseContent(instance_reference, hashed, content)
    def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
    software_url = self.getSoftwareURL()
    return self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference=partition_reference,
    partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
    shared=True
    )
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'testing partition 0',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'edgebot-1',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'text': 'edgebot-1',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    }
    ]
    def setUp(self):
    super().setUp()
    self.setUpMonitorConfigurationList()
    def test(self):
    # Note: Those tests do not run surykatka and do not do real checks, as
    # this depends too much on the environment and is really hard to
    # mock
    # So it is possible that some bugs might slip under the radar
    # Nevertheless the surykatka and check_surykatka_json are heavily
    # unit tested, and configuration created by the profiles is asserted
    # here, so it shall be enough as reasonable status
    self.requestEdgetestSlaves()
    self.initiateSurykatkaRun()
    self.assertSurykatkaStatusJSON()
    self.assertSurykatkaIni()
    self.assertSurykatkaBotPromise()
    self.assertSurykatkaPromises()
    self.assertSurykatkaCron()
    self.assertConnectionParameterDict()
    class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
    expected_connection_parameter_dict = {
    'active-region-list': ['1'],
    'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
    surykatka_dict = {
    'edge1': {
    1: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 3
    SQLITE = %(db_file)s
    URL =
    https://www.checkmaximumelapsedtime1.org/"""},
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.checkcertificateexpirationdays.org/
    https://www.checkfrontendiplist.org/
    https://www.checkhttpheaderdict.org/
    https://www.checkstatuscode.org/
    https://www.default.org/
    https://www.failureamount.org/"""},
    20: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 22
    SQLITE = %(db_file)s
    URL =
    https://www.checkmaximumelapsedtime20.org/"""},
    }
    }
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkcertificateexpirationdays',
    """extra_config_dict = { 'certificate-expiration-days': '20',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkcertificateexpirationdays.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkhttpheaderdict',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{"A": "AAA"}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkhttpheaderdict.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkmaximumelapsedtime1',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '1',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkmaximumelapsedtime1.org/'}""" % (
    self.surykatka_dict['edge1'][1]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkmaximumelapsedtime20',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '20',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkmaximumelapsedtime20.org/'}""" % (
    self.surykatka_dict['edge1'][20]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkstatuscode',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '300',
    'url': 'https://www.checkstatuscode.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'default',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.default.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'failureamount',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '10',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.failureamount.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkfrontendiplist',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '128.129.130.131 131.134.135.136',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkfrontendiplist.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'default',
    {'url': 'https://www.default.org/'},
    )
    self.requestEdgetestSlave(
    'checkstatuscode',
    {'url': 'https://www.checkstatuscode.org/', 'check-status-code': 300},
    )
    self.requestEdgetestSlave(
    'checkhttpheaderdict',
    {'url': 'https://www.checkhttpheaderdict.org/',
    'check-http-header-dict': {"A": "AAA"}},
    )
    self.requestEdgetestSlave(
    'checkcertificateexpirationdays',
    {'url': 'https://www.checkcertificateexpirationdays.org/',
    'check-certificate-expiration-days': '20'},
    )
    self.requestEdgetestSlave(
    'checkmaximumelapsedtime20',
    {'url': 'https://www.checkmaximumelapsedtime20.org/',
    'check-maximum-elapsed-time': 20},
    )
    self.requestEdgetestSlave(
    'checkmaximumelapsedtime1',
    {'url': 'https://www.checkmaximumelapsedtime1.org/',
    'check-maximum-elapsed-time': 1},
    )
    self.requestEdgetestSlave(
    'failureamount',
    {'url': 'https://www.failureamount.org/', 'failure-amount': '10'},
    )
    self.requestEdgetestSlave(
    'checkfrontendiplist',
    {'url': 'https://www.checkfrontendiplist.org/',
    'check-frontend-ip-list': ['128.129.130.131', '131.134.135.136']},
    )
    class TestEdgeNameserverListCheckFrontendIpList(
    EdgeSlaveMixin, SlapOSInstanceTestCase):
    expected_connection_parameter_dict = {
    'active-region-list': ['1'], 'sla-computer_guid': 'local',
    'sla-instance_guid': 'local-edge0'}
    surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    https://www.erp5.com/"""}
    }
    }
    @classmethod
    def getInstanceParameterDict(cls):
    return {'_': json.dumps({
    'nameserver-list': ['127.0.1.1', '127.0.1.2'],
    'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
    })}
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'ip-list': '127.0.0.1 127.0.0.2'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'report': 'http_query'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'status-code': '200'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'certificate-expiration-days': '15'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'url': 'https://www.erp5.com/'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'json-file': '%s'" % (self.surykatka_dict['edge1'][2]['json-file'],)
    )
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'failure-amount': '2'"
    )
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'backend',
    {'url': 'https://www.erp5.com/'},
    )
    class TestEdgeSlaveNotJson(
    EdgeSlaveMixin, SlapOSInstanceTestCase):
    surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.erp5.com/"""}
    }
    }
    # non-json provided in slave '_' results with damaging the cluster
    # test here is to expose real problem, which has no solution for now
    @unittest.expectedFailure
    def test(self):
    EdgeSlaveMixin.test()
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'default',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.default.org/'}""" % (
    self.surykatka_dict[2]['json-file'],))
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'default',
    {'url': 'https://www.default.org/'},
    )
    software_url = self.getSoftwareURL()
    self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference='notajson',
    partition_parameter_kw={'_': 'notajson'},
    shared=True
    )
    class TestEdgeRegion(EdgeSlaveMixin, SlapOSInstanceTestCase):
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Three',
    'title': 'edgebot-Region Three',
    'type': 'rss',
    'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    }
    ]
    @classmethod
    def setUpClassParameter(cls):
    cls.instance_parameter_dict = {
    'region-dict': {
    'Region One': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.1.1', '127.0.1.2'],
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    },
    'Region Two': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.2.1', '127.0.2.2'],
    },
    'Region Three': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    }
    }
    }
    cls.expected_connection_parameter_dict = {
    'active-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
    @classmethod
    def setUpClass(cls):
    cls.setUpClassParameter()
    super().setUpClass()
    def setUpParameter(self):
    self.surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    https://www.all.org/
    https://www.globalcheck.org/
    https://www.onetwo.org/
    https://www.specificcheck.org/
    https://www.specificoverride.org/"""},
    },
    'edge2': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.all.org/
    https://www.three.org/"""},
    },
    'edge3': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.2.1
    127.0.2.2
    URL =
    https://www.all.org/
    https://www.onetwo.org/
    https://www.parialmiss.org/
    https://www.specificoverride.org/"""},
    }
    }
    def setUp(self):
    self.setUpParameter()
    super().setUp()
    @classmethod
    def getInstanceParameterDict(cls):
    return {'_': json.dumps(cls.instance_parameter_dict)}
    slave_parameter_dict_dict = {
    'all': {
    'url': 'https://www.all.org/'
    },
    'onetwo': {
    'url': 'https://www.onetwo.org/',
    'region-dict': {'Region One': {}, 'Region Two': {}}
    },
    'three': {
    'url': 'https://www.three.org/',
    'region-dict': {'Region Three': {}}
    },
    'missed': {
    'url': 'https://www.missed.org/',
    'region-dict': {'Region Non Existing': {}}
    },
    'partialmiss': {
    'url': 'https://www.parialmiss.org/',
    'region-dict': {'Region Two': {}, 'Region Non Existing': {}}
    },
    'specificcheck': {
    'url': 'https://www.specificcheck.org/',
    'region-dict': {
    'Region One': {'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2']}}
    },
    'globalcheck': {
    'url': 'https://www.globalcheck.org/',
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'region-dict': {'Region One': {}}
    },
    'specificoverride': {
    'url': 'https://www.specificoverride.org/',
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'region-dict': {
    'Region One': {'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8']},
    'Region Two': {}}
    },
    }
    def requestEdgetestSlaves(self):
    for reference, parameter_dict in self.slave_parameter_dict_dict.items():
    self.requestEdgetestSlave(reference, parameter_dict)
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.1.3 127.0.1.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'specificcheck',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.1 99.99.99.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificcheck.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'globalcheck',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.3 99.99.99.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.globalcheck.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'specificoverride',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.7 99.99.99.8',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificoverride.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'onetwo',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.1.3 127.0.1.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.onetwo.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge2',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.3.1 127.0.3.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge2'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge2',
    'three',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.3.1 127.0.3.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.three.org/'}""" % (
    self.surykatka_dict['edge2'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'onetwo',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.onetwo.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'partialmiss',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.parialmiss.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'specificoverride',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.5 99.99.99.6',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificoverride.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    def test(self):
    super().test()
    self.assertSlaveConnectionParameterDict()
    maxDiff = None
    expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'specificoverride': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'three': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    }
    }
    },
    'globalcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'missed': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    def assertSlaveConnectionParameterDict(self):
    slave_connection_parameter_dict_dict = {}
    for reference, parameter_dict in self.slave_parameter_dict_dict.items():
    slave_connection_parameter_dict_dict[
    reference] = self.requestEdgetestSlave(
    reference, parameter_dict).getConnectionParameterDict()
    # unload the json
    slave_connection_parameter_dict_dict[
    reference] = json.loads(
    slave_connection_parameter_dict_dict[reference].pop('_'))
    self.assertEqual(
    self.expected_slave_connection_parameter_dict_dict,
    slave_connection_parameter_dict_dict
    )
    class TestEdgeRegionDestroyed(TestEdgeRegion):
    def setUpMonitorConfigurationList(self):
    # already for destroyed case, as test_monitor_setup will be called after
    # test
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    }
    ]
    def test(self):
    super().test()
    # hack around @classmethod
    self.__class__.instance_parameter_dict[
    'region-dict']['Region Three']['state'] = 'destroyed'
    # Region was removed
    self.__class__.expected_connection_parameter_dict[
    'active-region-list'].remove('Region Three')
    self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
    self.requestDefaultInstance()
    # give time to stabilise the tree
    self.slap.waitForInstance(max_retry=4)
    self.assertConnectionParameterDict()
    self.expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'specificoverride': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'three': {
    'assigned-region-dict': {},
    'available-region-list': [
    'Region One', 'Region Two'],
    },
    'globalcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'missed': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    self.assertSlaveConnectionParameterDict()
    class TestEdgeRegionAdded(TestEdgeRegion):
    def setUpMonitorConfigurationList(self):
    # already for added case, as test_monitor_setup will be called after test
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Four',
    'title': 'edgebot-Region Four',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Three',
    'title': 'edgebot-Region Three',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9704/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,)
    }
    ]
    def test(self):
    super().test()
    self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.4.1', '127.0.4.2'],
    'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
    }
    # Region was added
    self.__class__.expected_connection_parameter_dict[
    'active-region-list'].insert(0, 'Region Four')
    self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
    self.requestDefaultInstance()
    # give time to stabilise the tree, 6 times as new node is added
    self.slap.waitForInstance(max_retry=6)
    # XXX: few more times, but ignoring good result from promises, as there is
    # "Unknown Instance" of just added node which is not caught by any
    # promise, but in the end it shall get better
    for f in range(5):
    try:
    self.slap.waitForInstance()
    except Exception:
    pass
    self.slap.waitForInstance()
    self.assertConnectionParameterDict()
    self.expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Four': {
    'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
    'nameserver-list': ['127.0.4.1', '127.0.4.2']
    },
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    },
    'specificoverride': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    },
    'three': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    }
    }
    },
    'globalcheck': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    },
    'missed': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    self.assertSlaveConnectionParameterDict()
    self.surykatka_dict['edge4'] = {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(dbfile)
    NAMESERVER =
    127.0.4.1
    127.0.4.2
    URL =
    https://www.all.org/"""},
    }
    self.updateSurykatkaDict()
    self.assertHttpQueryPromiseContent(
    'edge4',
    'all',
    """{ 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.4.3 127.0.4.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge4'][2]['json-file'],))
    class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase): class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
    surykatka_dict = {} surykatka_dict = {}
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7