Skip to content

slapos
slapos
Commit ba7699ff authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

software/monitor: Drop obsoleted edgetest software type 

edgebot is dropped too, as it was shared instance implementation for edgetest.
parent b69af7c3
Pipeline #22893 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 1 addition and 1745 deletions
    software/monitor/README.edgetest.rst deleted 100644 → 0
    View file @ b69af7c3
    ============
    Edge testing
    ============
    ``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
    It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
    ``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
    ``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
    In order to monitor an url one need to:
    * request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
    * request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
    software/monitor/buildout.hash.cfg
    View file @ ba7699ff
    ......@@ -14,7 +14,7 @@
    # not need these here).
    [template]
    filename = instance.cfg
    md5sum = 0d26da3f83b293efbf330fd2b0350bf6
    md5sum = 072b276e0b0dd4b4a96a50348a04c7a7
    [template-monitor]
    _update_hash_filename_ = instance-monitor.cfg.jinja2
    ......@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
    _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
    md5sum = 8e0c4041f680312ff054687d7f28275a
    [template-monitor-edgetest]
    _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
    md5sum = a16fb139d9f3d01a88f499450fac50b4
    [template-monitor-edgebot]
    _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
    md5sum = 7b927af12934c228d2763c5a40c634b5
    [network-bench-cfg]
    filename = network_bench.cfg.in
    md5sum = 4f4d1a0afc225656278154e453587aea
    ......
    software/monitor/instance-edgetest-input-schema.json deleted 100644 → 0
    View file @ b69af7c3
    {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "properties": {
    "region-dict": {
    "title": "Regions",
    "description": "Defines regions of the cluster",
    "patternProperties": {
    ".*": {
    "properties": {
    "state": {
    "title": "State",
    "description": "State of the node of the region. Can be used to destroy not needed regions.",
    "type": "string",
    "default": "started",
    "enum": [
    "started",
    "stopped",
    "destroyed"
    ]
    },
    "sla-computer_guid": {
    "title": "GUID of the computer on which this region shall be deployed",
    "description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
    "type": "string",
    "default": ""
    },
    "nameserver-list": {
    "default": [],
    "title": "Nameservers",
    "description": "List of nameservers to use.",
    "type": "array"
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Default Frontend IPs to check",
    "description": "List of default frontend IPs to check, if empty no constraint is used.",
    "type": "array"
    }
    },
    "type": "object"
    }
    },
    "type": "object"
    },
    "nameserver-list": {
    "default": [],
    "title": "Nameservers (backward compatibility)",
    "description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
    "type": "array"
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Default Frontend IPs to check (backward compatibility)",
    "description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
    "type": "array"
    }
    }
    }
    software/monitor/instance-edgetest-slave-input-schema.json deleted 100644 → 0
    View file @ b69af7c3
    {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "properties": {
    "url": {
    "title": "URL to check",
    "description": "URL to check, like https://example.com/",
    "type": "string"
    },
    "region-dict": {
    "title": "Applicable Regions",
    "description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
    "patternProperties": {
    ".*": {
    "properties": {
    "state": {
    "title": "State",
    "description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
    "type": "string",
    "default": "present",
    "enum": [
    "present"
    ]
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Frontend IPs to check",
    "description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
    "type": "array"
    }
    },
    "type": "object"
    }
    },
    "type": "object",
    "default": {}
    },
    "check-status-code": {
    "title": "HTTP Code Check",
    "description": "Expected response HTTP Code.",
    "type": "number",
    "default": 200,
    "minimum": 100,
    "maximum": 599
    },
    "check-certificate-expiration-days": {
    "title": "Certificate Expiration Check (days)",
    "description": "Amount of days to consider certificate as being to-be-expired.",
    "type": "number",
    "default": 15,
    "minimum": 1
    },
    "check-maximum-elapsed-time": {
    "title": "Maximum Elapsed Check (seconds)",
    "description": "Maximum elapsed time for a site to reply to be considered good.",
    "type": "number",
    "default": 2,
    "minimum": 1
    },
    "check-http-header-dict": {
    "title": "HTTP Header Check",
    "description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
    "type": "object",
    "default": {}
    },
    "failure-amount": {
    "title": "Failure Amount",
    "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
    "type": "number",
    "default": 2,
    "minimum": 1
    },
    "check-frontend-ip-list": {
    "title": "Frontend IPs to check (backward compatibility)",
    "description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
    "type": "array"
    }
    }
    }
    software/monitor/instance-monitor-edgebot.cfg.jinja2 deleted 100644 → 0
    View file @ b69af7c3
    {%- if slap_software_type == software_type %}
    {%- set CONFIGURATION = {} %}
    {%- for k, v in sorted(slap_configuration.items()) %}
    {%- if k.startswith('configuration.') %}
    {%- do CONFIGURATION.__setitem__(k[14:], v) %}
    {%- endif %}
    {%- endfor %}
    {%- set slave_instance_dict = {} %}
    {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
    {%- if extra_slave_instance_list %}
    {#- Create slaves to process with setting up defaults #}
    {%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
    {%- do slave.setdefault('check-status-code', 200) %}
    {%- do slave.setdefault('check-http-header-dict', {}) %}
    {%- do slave.setdefault('check-certificate-expiration-days', 15) %}
    {%- do slave.setdefault('failure-amount', 2) %}
    {%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
    {%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
    {%- if 'url' in slave %}
    {%- set class = slave['check-maximum-elapsed-time'] %}
    {%- if class not in slave_instance_dict %}
    {%- do slave_instance_dict.__setitem__(class, []) %}
    {%- endif %}
    {%- do slave_instance_dict[class].append(slave) %}
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- set part_list = [] %}
    {%- for class, slave_instance_list in slave_instance_dict.items() %}
    {#- class is used to separate surykatka with different timeouts #}
    {%- set URL_LIST = [] %}
    {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
    {%- do URL_LIST.append(slave['url']) %}
    {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
    {%- do part_list.append(part_id) %}
    {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
    [{{part_id}}]
    <= monitor-promise-base
    promise = check_surykatka_json
    name = {{ safe_name }}.py
    config-report = http_query
    config-url = {{ slave['url'] }}
    config-status-code = {{ slave['check-status-code'] }}
    config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
    config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
    config-failure-amount = {{ slave['failure-amount'] }}
    config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
    config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
    config-json-file = ${surykatka-config-{{ class }}:json}
    {%- endfor %}
    [surykatka-bot-promise-{{ class }}]
    <= monitor-promise-base
    promise = check_surykatka_json
    name = surykatka-bot-promise-{{ class }}.py
    config-report = bot_status
    config-json-file = ${surykatka-config-{{ class }}:json}
    [surykatka-config-{{ class }}]
    recipe = slapos.recipe.template:jinja2
    db = ${directory:srv}/surykatka-{{ class }}.db
    output = ${directory:etc}/surykatka-{{ class }}.ini
    url = {{ template_surykatka_ini }}
    url_list = {{ dumps(URL_LIST) }}
    nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
    json = ${directory:srv}/surykatka-{{ class }}.json
    {#- timeout is just a bit bigger than class time #}
    timeout = {{ int(class) + 2 }}
    context =
    import json_module json
    key db :db
    key nameserver_list :nameserver_list
    key url_list :url_list
    key timeout :timeout
    [surykatka-{{ class }}]
    recipe = slapos.cookbook:wrapper
    config = ${surykatka-config-{{ class }}:output}
    command-line =
    {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
    wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
    hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
    [surykatka-status-json-{{ class }}]
    recipe = slapos.recipe.template
    json = ${surykatka-config-{{ class }}:json}
    inline =
    #!/bin/sh
    {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
    output = ${monitor-directory:bin}/${:_buildout_section_name_}
    [cron-entry-surykatka-status-{{ class }}]
    recipe = slapos.cookbook:cron.d
    cron-entries = ${directory:etc}/cron.d
    name = surykatka-status-{{ class }}
    frequency = */2 * * * *
    command = ${surykatka-status-json-{{ class }}:output}
    {%- do part_list.append('surykatka-%i'% (class,)) %}
    {%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
    {%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
    {%- endfor %}
    [buildout]
    extends = {{ monitor_template_output }}
    parts =
    cron
    monitor-base
    publish-connection-information
    {% for part_id in sorted(part_list) %}
    {{ part_id }}
    {% endfor %}
    eggs-directory = {{ eggs_directory }}
    develop-eggs-directory = {{ develop_eggs_directory }}
    offline = true
    [publish-connection-information]
    recipe = slapos.cookbook:publish.serialised
    monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
    monitor-url = ${monitor-publish-parameters:monitor-url}
    monitor-user = ${monitor-publish-parameters:monitor-user}
    monitor-password = ${monitor-publish-parameters:monitor-password}
    [monitor-instance-parameter]
    monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
    cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
    {% if slapparameter_dict.get('monitor-username', '') -%}
    username = {{ slapparameter_dict['monitor-username'] }}
    {% endif -%}
    {% if slapparameter_dict.get('monitor-password', '') -%}
    password = {{ slapparameter_dict['monitor-password'] }}
    {% endif -%}
    interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
    [monitor-directory]
    service = ${buildout:directory}/etc/service
    var = ${buildout:directory}/var
    srv = ${buildout:directory}/srv
    server-log = ${:private}/server-log
    monitor-log = ${:private}/monitor-log
    [slap-configuration]
    recipe = slapos.cookbook:slapconfiguration.serialised
    computer = ${slap-connection:computer-id}
    partition = ${slap-connection:partition-id}
    url = ${slap-connection:server-url}
    key = ${slap-connection:key-file}
    cert = ${slap-connection:cert-file}
    [slap-parameter]
    {%- endif %} {#- if slap_software_type == software_type #}
    software/monitor/instance-monitor-edgetest.cfg.jinja2 deleted 100644 → 0
    View file @ b69af7c3
    {%- if slap_software_type == software_type %}
    [jinja2-template-base]
    recipe = slapos.recipe.template:jinja2
    output = ${buildout:directory}/${:filename}
    extra-context =
    context =
    import json_module json
    ${:extra-context}
    [slave-test-configuration]
    <=jinja2-template-base
    url = {{ template_json_edgetest_test }}
    filename = srv/monitor/private/test.json
    extensions = jinja2.ext.do
    extra-context =
    section slave_information slap-configuration
    {% set part_list = [] -%}
    {%- set edgebot_software_type = 'edgebot' %}
    {%- set slave_list_name = 'extra_slave_instance_list' %}
    {%- set request_dict = {} %}
    {%- set namebase = "edgebot" %}
    {%- if 'region-dict' not in slapparameter_dict %}
    {#- Be nice and allow to work with default configuration #}
    {%- do slapparameter_dict.__setitem__('region-dict', {
    '1': {
    'sla-computer_guid': slap_configuration['computer'],
    'state': slap_configuration['instance-state'],
    'nameserver-list': slapparameter_dict.get('nameserver-list', []),
    'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
    }
    }) %}
    {%- endif %}
    {%- set active_region_list = [] %}
    {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
    {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
    {%- if region_parameter_dict.get('state', 'started') == 'started' %}
    {%- do active_region_list.append(region_name) %}
    {%- endif %}
    {%- endfor %}
    {%- set authorized_slave_dict = {} %}
    {%- set publish_slave_dict_dict = {} %}
    {%- for slave in slave_instance_list | sort(attribute='slave_title') %}
    {%- set slave_reference = slave.pop('slave_reference') %}
    {%- set publish_dict = {'assigned-region-dict': {}} %}
    {%- if '_' in slave %}
    {%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
    {%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
    {%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
    {%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
    {%- if slave_region_dict == {} %}
    {%- for region in active_region_list %}
    {%- do slave_region_dict.__setitem__(region, {}) %}
    {%- endfor %}
    {%- endif %}
    {%- for region in slave_region_dict %}
    {%- if region in active_region_list %}
    {%- set region_info = {
    'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
    'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
    } %}
    {%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
    {%- set slave_dict = base_slave_dict.copy() %}
    {%- do slave_dict.update(region_info) %}
    {%- if region not in authorized_slave_dict %}
    {%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
    {%- else %}
    {%- do authorized_slave_dict[region].append(slave_dict) %}
    {%- endif %}
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
    {%- endfor %}
    {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
    {%- set number = {'i': 1} %}
    {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
    {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
    {%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
    {%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
    {%- do part_list.append(request_section_title) %}
    {#- Note: monitor-httpd-port will vary on regions being added and removed,
    but this is accepted, as it's only internal trick #}
    {%- do request_dict.__setitem__(
    request_section_title,
    {
    'config': {
    'monitor-httpd-port': monitor_base_port + number['i'],
    'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
    'nameserver-list': region_parameter_dict.get('nameserver-list', []),
    'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
    },
    'name': edgebot_name,
    'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
    'state': region_parameter_dict.get('state', 'started'),
    }) %}
    {%- do number.__setitem__('i', number['i'] + 1) %}
    {%- endfor %}
    [replicate]
    <= slap-connection
    recipe = slapos.cookbook:request.serialised
    config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
    config-monitor-username = ${monitor-instance-parameter:username}
    config-monitor-password = ${monitor-htpasswd:passwd}
    software-url = ${slap-connection:software-release-url}
    software-type = {{edgebot_software_type}}
    {%- set monitor_base_url_dict = {} -%}
    {% for section, edgebot_request in request_dict.items() %}
    [{{section}}]
    <= replicate
    name = {{ edgebot_request['name'] }}
    state = {{ edgebot_request['state'] }}
    {%- if edgebot_request['state'] != 'destroyed' %}
    {%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
    return = monitor-base-url
    {%- endif %}
    {%- set edgebot_configuration_dict = edgebot_request['config'] %}
    {%- for config_key, config_value in edgebot_configuration_dict.items() %}
    config-{{ config_key }} = {{ dumps(config_value) }}
    {% endfor -%}
    {%- for parameter, value in edgebot_request['sla'].items() %}
    sla-{{ parameter }} = {{ value }}
    {%- endfor %}
    {%- endfor %}
    # Publish information for each slave
    {%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
    {%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
    {%- do part_list.append(publish_section_title) %}
    [{{ publish_section_title }}]
    recipe = slapos.cookbook:publish.serialised
    available-region-list = {{ dumps(list(active_region_list)) }}
    -slave-reference = {{ slave_reference }}
    {%- for key, value in publish_dict.items() %}
    {{ key }} = {{ dumps(value) }}
    {%- endfor %}
    {% endfor %}
    [monitor-conf-parameters]
    monitor-title = Monitor
    password = ${monitor-htpasswd:passwd}
    [monitor-base-url-dict]
    {% for key, value in monitor_base_url_dict.items() -%}
    {{ key }} = {{ value }}
    {% endfor %}
    [buildout]
    extends = {{ instance_base_monitor }}
    parts +=
    slave-test-configuration
    {% for part in part_list %}
    {{ ' %s' % part }}
    {%- endfor %}
    [publish-connection-information]
    recipe = slapos.cookbook:publish.serialised
    active-region-list = {{ dumps(list(active_region_list)) }}
    sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
    sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
    {%- endif %} {#- if slap_software_type == software_type #}
    software/monitor/instance.cfg
    View file @ ba7699ff
    ......@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
    [switch_softwaretype]
    recipe = slapos.cookbook:switch-softwaretype
    default = instance-base-monitor:output
    edgetest = instance-base-edgetest:output
    edgebot = instance-base-edgebot:output
    edgetest-basic = instance-edgetest-basic:output
    RootSoftwareInstance = $${:default}
    ......@@ -54,45 +52,6 @@ extra-context =
    key template_surykatka_ini surykatka:ini
    key surykatka_binary surykatka:binary
    [instance-base-edgetest]
    recipe = slapos.recipe.template:jinja2
    url = ${template-monitor-edgetest:target}
    output = $${buildout:directory}/template-monitor-base-edgetest.cfg
    extensions = jinja2.ext.do
    context = import json_module json
    import hashlib_module hashlib
    key develop_eggs_directory buildout:develop-eggs-directory
    key eggs_directory buildout:eggs-directory
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    section slap_configuration slap-configuration
    raw software_type edgetest
    key instance_base_monitor instance-base-monitor:output
    key slave_instance_list slap-configuration:slave-instance-list
    raw buildout_bin ${buildout:bin-directory}
    raw template_json_edgetest_test ${json-test-template:target}
    [instance-base-edgebot]
    recipe = slapos.recipe.template:jinja2
    url = ${template-monitor-edgebot:target}
    output = $${buildout:directory}/template-monitor-edgebot.cfg
    extensions = jinja2.ext.do
    context = import json_module json
    import hashlib_module hashlib
    key develop_eggs_directory buildout:develop-eggs-directory
    key eggs_directory buildout:eggs-directory
    section slap_configuration slap-configuration
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    raw software_type edgebot
    key surykatka_binary surykatka:binary
    key template_surykatka_ini surykatka:ini
    raw buildout_bin ${buildout:bin-directory}
    raw monitor_template_output ${monitor-template:output}
    raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
    [slap-configuration]
    recipe = slapos.cookbook:slapconfiguration.serialised
    computer = $${slap-connection:computer-id}
    ......
    software/monitor/software.cfg
    View file @ ba7699ff
    ......@@ -13,7 +13,6 @@ parts =
    network-bench-cfg
    json-test-template
    template
    template-monitor-edgetest
    template-monitor
    [python]
    ......@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-monitor-edgetest]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-monitor-edgebot]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    [template-surykatka-ini]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    ......
    software/monitor/software.cfg.json
    View file @ ba7699ff
    ......@@ -17,24 +17,6 @@
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 1
    },
    "edgetest": {
    "title": "Edge Test [OBSOLETE]",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 2
    },
    "edgetest-slave": {
    "title": "Edge Test Slave [OBSOLETE]",
    "shared": true,
    "software-type": "edgetest",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-slave-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 3
    }
    }
    }
    software/monitor/test/test.py
    View file @ ba7699ff
    ......@@ -32,7 +32,6 @@ import os
    import re
    import requests
    import subprocess
    import unittest
    import xml.etree.ElementTree as ET
    from slapos.recipe.librecipe import generateHashFromFiles
    from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
    ......@@ -217,1204 +216,6 @@ class EdgeMixin(object):
    self.assertIn('bot_status', status_json)
    class EdgeSlaveMixin(EdgeMixin, MonitorTestMixin):
    @classmethod
    def setUpClass(cls):
    # XXX we run these tests with --all as a workaround for the fact that after
    # requesting new shared instances we don't have promise to wait for the
    # processing of these shared instances to be completed.
    # The sequence is something like this:
    # - `requestEdgetestSlaves` will request edgetest partition
    # - first `waitForInstance` will process the edgetest partition, which
    # will request a edgebot partition, but without promise to wait for the
    # processing to be finished, so the first run of `slapos node instance`
    # exits with success code and `waitForInstance` return.
    # - second `waitForInstance` process the edgebot partition.
    # Once we implement a promise (or something similar) here, we should not
    # have to use --all
    cls.slap._force_slapos_node_instance_all = True
    return super().setUpClass()
    def assertConnectionParameterDict(self):
    serialised = self.requestDefaultInstance().getConnectionParameterDict()
    connection_parameter_dict = json.loads(serialised['_'])
    # tested elsewhere
    connection_parameter_dict.pop('monitor-setup-url', None)
    # comes from instance-monitor.cfg.jinja2, not needed here
    connection_parameter_dict.pop('server_log_url', None)
    self.assertEqual(
    self.expected_connection_parameter_dict,
    connection_parameter_dict
    )
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'edgetest'
    def updateSurykatkaDict(self):
    for instance_reference in self.surykatka_dict:
    for class_ in self.surykatka_dict[instance_reference]:
    update_dict = {}
    update_dict['ini-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'surykatka-%s.ini' % (class_,))
    update_dict['json-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.json' % (class_,))
    update_dict['status-json'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'bin',
    'surykatka-status-json-%s' % (class_,))
    update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
    update_dict['status-cron'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'cron.d', 'surykatka-status-%s' % (class_,))
    update_dict['db_file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.db' % (class_,))
    self.surykatka_dict[instance_reference][class_].update(update_dict)
    def assertHttpQueryPromiseContent(self, instance_reference, name, content):
    hashed = 'http-query-%s-promise.py' % (
    hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
    self.assertPromiseContent(instance_reference, hashed, content)
    def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
    software_url = self.getSoftwareURL()
    return self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference=partition_reference,
    partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
    shared=True
    )
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'testing partition 0',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'edgebot-1',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'text': 'edgebot-1',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    }
    ]
    def setUp(self):
    super().setUp()
    self.setUpMonitorConfigurationList()
    def test(self):
    # Note: Those tests do not run surykatka and do not do real checks, as
    # this depends too much on the environment and is really hard to
    # mock
    # So it is possible that some bugs might slip under the radar
    # Nevertheless the surykatka and check_surykatka_json are heavily
    # unit tested, and configuration created by the profiles is asserted
    # here, so it shall be enough as reasonable status
    self.requestEdgetestSlaves()
    self.initiateSurykatkaRun()
    self.assertSurykatkaStatusJSON()
    self.assertSurykatkaIni()
    self.assertSurykatkaBotPromise()
    self.assertSurykatkaPromises()
    self.assertSurykatkaCron()
    self.assertConnectionParameterDict()
    class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
    expected_connection_parameter_dict = {
    'active-region-list': ['1'],
    'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
    surykatka_dict = {
    'edge1': {
    1: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 3
    SQLITE = %(db_file)s
    URL =
    https://www.checkmaximumelapsedtime1.org/"""},
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.checkcertificateexpirationdays.org/
    https://www.checkfrontendiplist.org/
    https://www.checkhttpheaderdict.org/
    https://www.checkstatuscode.org/
    https://www.default.org/
    https://www.failureamount.org/"""},
    20: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 22
    SQLITE = %(db_file)s
    URL =
    https://www.checkmaximumelapsedtime20.org/"""},
    }
    }
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkcertificateexpirationdays',
    """extra_config_dict = { 'certificate-expiration-days': '20',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkcertificateexpirationdays.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkhttpheaderdict',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{"A": "AAA"}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkhttpheaderdict.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkmaximumelapsedtime1',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '1',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkmaximumelapsedtime1.org/'}""" % (
    self.surykatka_dict['edge1'][1]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkmaximumelapsedtime20',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '20',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkmaximumelapsedtime20.org/'}""" % (
    self.surykatka_dict['edge1'][20]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkstatuscode',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '300',
    'url': 'https://www.checkstatuscode.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'default',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.default.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'failureamount',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '10',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.failureamount.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'checkfrontendiplist',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '128.129.130.131 131.134.135.136',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.checkfrontendiplist.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'default',
    {'url': 'https://www.default.org/'},
    )
    self.requestEdgetestSlave(
    'checkstatuscode',
    {'url': 'https://www.checkstatuscode.org/', 'check-status-code': 300},
    )
    self.requestEdgetestSlave(
    'checkhttpheaderdict',
    {'url': 'https://www.checkhttpheaderdict.org/',
    'check-http-header-dict': {"A": "AAA"}},
    )
    self.requestEdgetestSlave(
    'checkcertificateexpirationdays',
    {'url': 'https://www.checkcertificateexpirationdays.org/',
    'check-certificate-expiration-days': '20'},
    )
    self.requestEdgetestSlave(
    'checkmaximumelapsedtime20',
    {'url': 'https://www.checkmaximumelapsedtime20.org/',
    'check-maximum-elapsed-time': 20},
    )
    self.requestEdgetestSlave(
    'checkmaximumelapsedtime1',
    {'url': 'https://www.checkmaximumelapsedtime1.org/',
    'check-maximum-elapsed-time': 1},
    )
    self.requestEdgetestSlave(
    'failureamount',
    {'url': 'https://www.failureamount.org/', 'failure-amount': '10'},
    )
    self.requestEdgetestSlave(
    'checkfrontendiplist',
    {'url': 'https://www.checkfrontendiplist.org/',
    'check-frontend-ip-list': ['128.129.130.131', '131.134.135.136']},
    )
    class TestEdgeNameserverListCheckFrontendIpList(
    EdgeSlaveMixin, SlapOSInstanceTestCase):
    expected_connection_parameter_dict = {
    'active-region-list': ['1'], 'sla-computer_guid': 'local',
    'sla-instance_guid': 'local-edge0'}
    surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    https://www.erp5.com/"""}
    }
    }
    @classmethod
    def getInstanceParameterDict(cls):
    return {'_': json.dumps({
    'nameserver-list': ['127.0.1.1', '127.0.1.2'],
    'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
    })}
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'ip-list': '127.0.0.1 127.0.0.2'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'report': 'http_query'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'status-code': '200'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'certificate-expiration-days': '15'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'url': 'https://www.erp5.com/'")
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'json-file': '%s'" % (self.surykatka_dict['edge1'][2]['json-file'],)
    )
    self.assertHttpQueryPromiseContent(
    'edge1',
    'backend',
    "'failure-amount': '2'"
    )
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'backend',
    {'url': 'https://www.erp5.com/'},
    )
    class TestEdgeSlaveNotJson(
    EdgeSlaveMixin, SlapOSInstanceTestCase):
    surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.erp5.com/"""}
    }
    }
    # non-json provided in slave '_' results with damaging the cluster
    # test here is to expose real problem, which has no solution for now
    @unittest.expectedFailure
    def test(self):
    EdgeSlaveMixin.test()
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'default',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.default.org/'}""" % (
    self.surykatka_dict[2]['json-file'],))
    def requestEdgetestSlaves(self):
    self.requestEdgetestSlave(
    'default',
    {'url': 'https://www.default.org/'},
    )
    software_url = self.getSoftwareURL()
    self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference='notajson',
    partition_parameter_kw={'_': 'notajson'},
    shared=True
    )
    class TestEdgeRegion(EdgeSlaveMixin, SlapOSInstanceTestCase):
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Three',
    'title': 'edgebot-Region Three',
    'type': 'rss',
    'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    }
    ]
    @classmethod
    def setUpClassParameter(cls):
    cls.instance_parameter_dict = {
    'region-dict': {
    'Region One': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.1.1', '127.0.1.2'],
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    },
    'Region Two': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.2.1', '127.0.2.2'],
    },
    'Region Three': {
    'sla-computer_guid': 'local',
    'state': 'started',
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    }
    }
    }
    cls.expected_connection_parameter_dict = {
    'active-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'sla-computer_guid': 'local', 'sla-instance_guid': 'local-edge0'}
    @classmethod
    def setUpClass(cls):
    cls.setUpClassParameter()
    super().setUpClass()
    def setUpParameter(self):
    self.surykatka_dict = {
    'edge1': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    https://www.all.org/
    https://www.globalcheck.org/
    https://www.onetwo.org/
    https://www.specificcheck.org/
    https://www.specificoverride.org/"""},
    },
    'edge2': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    URL =
    https://www.all.org/
    https://www.three.org/"""},
    },
    'edge3': {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.2.1
    127.0.2.2
    URL =
    https://www.all.org/
    https://www.onetwo.org/
    https://www.parialmiss.org/
    https://www.specificoverride.org/"""},
    }
    }
    def setUp(self):
    self.setUpParameter()
    super().setUp()
    @classmethod
    def getInstanceParameterDict(cls):
    return {'_': json.dumps(cls.instance_parameter_dict)}
    slave_parameter_dict_dict = {
    'all': {
    'url': 'https://www.all.org/'
    },
    'onetwo': {
    'url': 'https://www.onetwo.org/',
    'region-dict': {'Region One': {}, 'Region Two': {}}
    },
    'three': {
    'url': 'https://www.three.org/',
    'region-dict': {'Region Three': {}}
    },
    'missed': {
    'url': 'https://www.missed.org/',
    'region-dict': {'Region Non Existing': {}}
    },
    'partialmiss': {
    'url': 'https://www.parialmiss.org/',
    'region-dict': {'Region Two': {}, 'Region Non Existing': {}}
    },
    'specificcheck': {
    'url': 'https://www.specificcheck.org/',
    'region-dict': {
    'Region One': {'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2']}}
    },
    'globalcheck': {
    'url': 'https://www.globalcheck.org/',
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'region-dict': {'Region One': {}}
    },
    'specificoverride': {
    'url': 'https://www.specificoverride.org/',
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'region-dict': {
    'Region One': {'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8']},
    'Region Two': {}}
    },
    }
    def requestEdgetestSlaves(self):
    for reference, parameter_dict in self.slave_parameter_dict_dict.items():
    self.requestEdgetestSlave(reference, parameter_dict)
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge1',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.1.3 127.0.1.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'specificcheck',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.1 99.99.99.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificcheck.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'globalcheck',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.3 99.99.99.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.globalcheck.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'specificoverride',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.7 99.99.99.8',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificoverride.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge1',
    'onetwo',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.1.3 127.0.1.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.onetwo.org/'}""" % (
    self.surykatka_dict['edge1'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge2',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.3.1 127.0.3.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge2'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge2',
    'three',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.3.1 127.0.3.2',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.three.org/'}""" % (
    self.surykatka_dict['edge2'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'all',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'onetwo',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.onetwo.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'partialmiss',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.parialmiss.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge3',
    'specificoverride',
    """extra_config_dict = { 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '99.99.99.5 99.99.99.6',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.specificoverride.org/'}""" % (
    self.surykatka_dict['edge3'][2]['json-file'],))
    def test(self):
    super().test()
    self.assertSlaveConnectionParameterDict()
    maxDiff = None
    expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'specificoverride': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'three': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    }
    }
    },
    'globalcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two']
    },
    'missed': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    def assertSlaveConnectionParameterDict(self):
    slave_connection_parameter_dict_dict = {}
    for reference, parameter_dict in self.slave_parameter_dict_dict.items():
    slave_connection_parameter_dict_dict[
    reference] = self.requestEdgetestSlave(
    reference, parameter_dict).getConnectionParameterDict()
    # unload the json
    slave_connection_parameter_dict_dict[
    reference] = json.loads(
    slave_connection_parameter_dict_dict[reference].pop('_'))
    self.assertEqual(
    self.expected_slave_connection_parameter_dict_dict,
    slave_connection_parameter_dict_dict
    )
    class TestEdgeRegionDestroyed(TestEdgeRegion):
    def setUpMonitorConfigurationList(self):
    # already for destroyed case, as test_monitor_setup will be called after
    # test
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    }
    ]
    def test(self):
    super().test()
    # hack around @classmethod
    self.__class__.instance_parameter_dict[
    'region-dict']['Region Three']['state'] = 'destroyed'
    # Region was removed
    self.__class__.expected_connection_parameter_dict[
    'active-region-list'].remove('Region Three')
    self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
    self.requestDefaultInstance()
    # give time to stabilise the tree
    self.slap.waitForInstance(max_retry=4)
    self.assertConnectionParameterDict()
    self.expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'specificoverride': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'three': {
    'assigned-region-dict': {},
    'available-region-list': [
    'Region One', 'Region Two'],
    },
    'globalcheck': {
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    'available-region-list': [
    'Region One', 'Region Two']
    },
    'missed': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region One', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    self.assertSlaveConnectionParameterDict()
    class TestEdgeRegionAdded(TestEdgeRegion):
    def setUpMonitorConfigurationList(self):
    # already for added case, as test_monitor_setup will be called after test
    self.monitor_configuration_list = [
    {
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'title': 'testing partition 0',
    'type': 'rss',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Four',
    'title': 'edgebot-Region Four',
    'type': 'rss',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region One',
    'title': 'edgebot-Region One',
    'type': 'rss',
    'url': 'https://[%s]:9702/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9702/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Three',
    'title': 'edgebot-Region Three',
    'type': 'rss',
    'url': 'https://[%s]:9703/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9703/public/feed' % (self._ipv6_address,)
    },
    {
    'htmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,),
    'text': 'edgebot-Region Two',
    'title': 'edgebot-Region Two',
    'type': 'rss',
    'url': 'https://[%s]:9704/share/private/' % (self._ipv6_address,),
    'version': 'RSS',
    'xmlUrl': 'https://[%s]:9704/public/feed' % (self._ipv6_address,)
    }
    ]
    def test(self):
    super().test()
    self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
    'sla-computer_guid': 'local',
    'state': 'started',
    'nameserver-list': ['127.0.4.1', '127.0.4.2'],
    'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
    }
    # Region was added
    self.__class__.expected_connection_parameter_dict[
    'active-region-list'].insert(0, 'Region Four')
    self.__class__._instance_parameter_dict = self.getInstanceParameterDict()
    self.requestDefaultInstance()
    # give time to stabilise the tree, 6 times as new node is added
    self.slap.waitForInstance(max_retry=6)
    # XXX: few more times, but ignoring good result from promises, as there is
    # "Unknown Instance" of just added node which is not caught by any
    # promise, but in the end it shall get better
    for f in range(5):
    try:
    self.slap.waitForInstance()
    except Exception:
    pass
    self.slap.waitForInstance()
    self.assertConnectionParameterDict()
    self.expected_slave_connection_parameter_dict_dict = {
    'all': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Four': {
    'check-frontend-ip-list': ['127.0.4.3', '127.0.4.4'],
    'nameserver-list': ['127.0.4.1', '127.0.4.2']
    },
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'onetwo': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['127.0.1.3', '127.0.1.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    },
    'specificcheck': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.1', '99.99.99.2'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    },
    'specificoverride': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.7', '99.99.99.8'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    },
    'Region Two': {
    'check-frontend-ip-list': ['99.99.99.5', '99.99.99.6'],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    },
    },
    'three': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Three': {
    'check-frontend-ip-list': ['127.0.3.1', '127.0.3.2'],
    'nameserver-list': []
    }
    }
    },
    'globalcheck': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region One': {
    'check-frontend-ip-list': ['99.99.99.3', '99.99.99.4'],
    'nameserver-list': ['127.0.1.1', '127.0.1.2']
    }
    },
    },
    'missed': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    }
    },
    'partialmiss': {
    'available-region-list': [
    'Region Four', 'Region One', 'Region Three', 'Region Two'],
    'assigned-region-dict': {
    'Region Two': {
    'check-frontend-ip-list': [],
    'nameserver-list': ['127.0.2.1', '127.0.2.2']
    }
    }
    }
    }
    self.assertSlaveConnectionParameterDict()
    self.surykatka_dict['edge4'] = {
    2: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 4
    SQLITE = %(dbfile)
    NAMESERVER =
    127.0.4.1
    127.0.4.2
    URL =
    https://www.all.org/"""},
    }
    self.updateSurykatkaDict()
    self.assertHttpQueryPromiseContent(
    'edge4',
    'all',
    """{ 'certificate-expiration-days': '15',
    'failure-amount': '2',
    'http-header-dict': '{}',
    'ip-list': '127.0.4.3 127.0.4.4',
    'json-file': '%s',
    'maximum-elapsed-time': '2',
    'report': 'http_query',
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge4'][2]['json-file'],))
    class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
    surykatka_dict = {}
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7