diff --git a/component/surykatka/buildout.cfg b/component/surykatka/buildout.cfg deleted file mode 100644 index d61c64c7873cb8c5f597cf900b93205b232f2d1c..0000000000000000000000000000000000000000 --- a/component/surykatka/buildout.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[buildout] -extends = - buildout.hash.cfg - ../python3/buildout.cfg - -parts = - surykatka - -[surykatka-requirements] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/${:_update_hash_filename_} - -[surykatka] -recipe = plone.recipe.command -command = - bash -c "${python3.7:executable} -m venv ${:location} && \ - . ${:location}/bin/activate && \ - pip install -r ${surykatka-requirements:target}" -location = ${buildout:parts-directory}/${:_buildout_section_name_} -stop-on-error = true -executable = ${:location}/bin/surykatka diff --git a/component/surykatka/buildout.hash.cfg b/component/surykatka/buildout.hash.cfg deleted file mode 100644 index 6dc5bac2174d30c5274936825cecfedb444193a3..0000000000000000000000000000000000000000 --- a/component/surykatka/buildout.hash.cfg +++ /dev/null @@ -1,17 +0,0 @@ -# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax. -# The only allowed lines here are (regexes): -# - "^#" comments, copied verbatim -# - "^[" section beginings, copied verbatim -# - lines containing an "=" sign which must fit in the following categorie. -# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file -# Copied verbatim. -# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported -# by the re-generation script. -# Re-generated. -# - other lines are copied verbatim -# Substitution (${...:...}), extension ([buildout] extends = ...) and -# section inheritance (< = ...) are NOT supported (but you should really -# not need these here). -[surykatka-requirements] -_update_hash_filename_ = requirements.txt -md5sum = c4bd35bcc5c9c25efe1edff5dd022605 diff --git a/component/surykatka/requirements.txt b/component/surykatka/requirements.txt deleted file mode 100644 index 4acc4a4f6e451883504fbffe8f42e3ec95252950..0000000000000000000000000000000000000000 --- a/component/surykatka/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -Click==7.0 -certifi==2019.11.28 -chardet==3.0.4 -dnspython==1.16.0 -forcediphttpsadapter==1.0.1 -idna==2.8 -miniupnpc==2.0.2 -peewee==3.13.1 -requests==2.22.0 -surykatka==0.4.2 -urllib3==1.25.8 diff --git a/software/monitor/buildout.hash.cfg b/software/monitor/buildout.hash.cfg index a0167666aa63c193365ff8b27b87edda2aa91680..f405fd1acad2e7c54aecf85abb38f4b4e61738d0 100644 --- a/software/monitor/buildout.hash.cfg +++ b/software/monitor/buildout.hash.cfg @@ -14,7 +14,7 @@ # not need these here). [template] filename = instance.cfg -md5sum = dc9770bacea2c504b92ad2162e58d222 +md5sum = 07873511a15df74f0be761947819ac9b [template-monitor] _update_hash_filename_ = instance-monitor.cfg.jinja2 @@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 [template-monitor-edgetest] _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2 -md5sum = 9e237dbdda59e788202f0da194a57d41 +md5sum = a57106ee88ff3295b9ffce84105da79b [template-monitor-edgebot] _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 -md5sum = f28a329e830ed737d468abcb4e89e1a2 +md5sum = c535f4df6388cdc3f6c1df2a91d6dd53 [network-bench-cfg] filename = network_bench.cfg.in @@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 [monitor-collect-csv-dump] filename = script/collect_csv_dump.py -md5sum = cad2402bbd21907cfed6bc5af8c5d3ab +md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb [template-surykatka-ini] _update_hash_filename_ = surykatka.ini.jinja2 -md5sum = a2de719a5a65438c8c3ee5195442beb6 +md5sum = 89545501f0e5bf11608978886429da3d diff --git a/software/monitor/instance-monitor-edgebot.cfg.jinja2 b/software/monitor/instance-monitor-edgebot.cfg.jinja2 index 1ba34633cc68d1a8953967e892df8ca2a7dbd26b..f499e419be3ccebeae1a3681f791f562583655db 100644 --- a/software/monitor/instance-monitor-edgebot.cfg.jinja2 +++ b/software/monitor/instance-monitor-edgebot.cfg.jinja2 @@ -9,7 +9,7 @@ {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %} {%- if extra_slave_instance_list %} {#- Create slaves to process with setting up defaults #} -{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %} +{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %} {%- if 'check-status-code' not in slave %} {%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %} {%- endif %} @@ -37,7 +37,7 @@ {%- set part_list = [] %} {%- for class, slave_instance_list in slave_instance_dict.items() %} {#- class is used to separate surykatka with different timeouts #} -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %} {%- do part_list.append(part_id) %} {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %} diff --git a/software/monitor/instance-monitor-edgetest.cfg.jinja2 b/software/monitor/instance-monitor-edgetest.cfg.jinja2 index b3e147cd6a10c27d75aca5e8de7091e681188d7c..0b628af460783186f664631380f6a7192fafee14 100644 --- a/software/monitor/instance-monitor-edgetest.cfg.jinja2 +++ b/software/monitor/instance-monitor-edgetest.cfg.jinja2 @@ -27,7 +27,7 @@ extra-context = {%- set namebase = "edgebot" %} {%- set authorized_slave_list = [] %} {%- set monitor_base_url_dict = {} -%} -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- do authorized_slave_list.append(slave) %} {%- endfor %} {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %} @@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url} software-type = {{edgebot_software_type}} return = monitor-base-url -{% for section, edgebot_request in request_dict.iteritems() %} +{% for section, edgebot_request in request_dict.items() %} [{{section}}] <= replicate name = {{ edgebot_request.get('name') }} @@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }} {%- set slave_configuration_dict = slapparameter_dict %} {%- do slave_configuration_dict.update(edgebot_request.get('config')) %} {%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %} -{%- for config_key, config_value in slave_configuration_dict.iteritems() %} +{%- for config_key, config_value in slave_configuration_dict.items() %} config-{{ config_key }} = {{ dumps(config_value) }} {% endfor -%} {%- if edgebot_request.get('sla') %} -{%- for parameter, value in edgebot_request.get('sla').iteritems() %} +{%- for parameter, value in edgebot_request.get('sla').items() %} sla-{{ parameter }} = {{ value }} {%- endfor %} {%- else %} diff --git a/software/monitor/instance.cfg b/software/monitor/instance.cfg index a11a2bb744a067553b1d56be2c518cfbe7fbd0fb..29f197453f38c88aceeeb982db34aa911ccc0b1e 100644 --- a/software/monitor/instance.cfg +++ b/software/monitor/instance.cfg @@ -44,17 +44,12 @@ context = import json_module json raw template_json_edgetest_test ${json-test-template:target} mode = 0644 -[surykatka-wrapped] -recipe = slapos.cookbook:wrapper -wrapper-path = $${buildout:bin-directory}/surykatka-wrapped -command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable} - [instance-base-edgebot] recipe = slapos.recipe.template:jinja2 template = ${template-monitor-edgebot:target} rendered = $${buildout:directory}/template-monitor-edgebot.cfg extensions = jinja2.ext.do -surykatka-binary = ${surykatka:executable} +surykatka-binary = ${buildout:bin-directory}/surykatka template-surykatka-ini = ${template-surykatka-ini:target} context = import json_module json @@ -64,7 +59,7 @@ context = import json_module json key slapparameter_dict slap-configuration:configuration key slap_software_type slap-configuration:slap-software-type raw software_type edgebot - key surykatka_binary surykatka-wrapped:wrapper-path + key surykatka_binary :surykatka-binary key template_surykatka_ini :template-surykatka-ini raw buildout_bin ${buildout:bin-directory} raw monitor_template_output ${monitor-template:output} diff --git a/software/monitor/script/collect_csv_dump.py b/software/monitor/script/collect_csv_dump.py index 085176a9b19ed726cef75b45537ad156ea2d1ad6..607bdd61d438ed2af45f18d2c56db0101bb4e098 100644 --- a/software/monitor/script/collect_csv_dump.py +++ b/software/monitor/script/collect_csv_dump.py @@ -27,6 +27,7 @@ # ############################################################################## +from __future__ import print_function import os import argparse import csv @@ -53,15 +54,14 @@ def parseArguments(): return parser.parse_args() def writeFile(name, folder, date_scope, rows): - if os.path.exists( - os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))): + folder = os.path.join(folder, date_scope) + f = os.path.join(folder, "dump_%s.csv" % name) + if os.path.exists(f): # File already exists, no reason to recreate it. return - mkdir_p(os.path.join(folder, date_scope), 0o755) - file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w") - csv_output = csv.writer(file_io) - csv_output.writerows(rows) - file_io.close() + mkdir_p(folder, 0o755) + with open(f, "w") as file_io: + csv.writer(file_io).writerows(rows) def dump_table_into_csv(db, folder): db.connect() @@ -96,6 +96,6 @@ if __name__ == "__main__": if not os.path.exists(parser.collector_db): - print "Collector database not found..." + print("Collector database not found...") dump_table_into_csv(Database(parser.collector_db), parser.output_folder) diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index 270e70311ae5108fa761f6e8de8eda5f63936745..b88fb7d2ed4a5bfbedbc8d63de38555b4675e822 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -4,9 +4,11 @@ extends = buildout.hash.cfg ../../component/pycurl/buildout.cfg ../../component/python-cryptography/buildout.cfg - ../../component/surykatka/buildout.cfg ../../stack/monitor/buildout.cfg ../../stack/slapos.cfg + ../../component/python3/buildout.cfg + +python = python3 parts = slapos-cookbook @@ -68,6 +70,7 @@ eggs += ${python-cryptography:egg} slapos.toolbox slapos.core + surykatka scripts = networkbench @@ -78,6 +81,7 @@ scripts = monitor.genstatus monitor.configwrite check-computer-memory + surykatka [monitor-eggs] eggs += @@ -88,3 +92,17 @@ eggs += [versions] slapos.recipe.template = 4.4 + +surykatka = 0.4.2 + +# For surykatka 0.4.2 +click = 7.0 +certifi = 2019.11.28 +chardet = 3.0.4 +dnspython = 1.16.0 +forcediphttpsadapter = 1.0.1 +idna = 2.8 +miniupnpc = 2.0.2 +peewee = 3.13.1 +requests = 2.22.0 +urllib3 = 1.25.8 diff --git a/software/monitor/surykatka.ini.jinja2 b/software/monitor/surykatka.ini.jinja2 index 788fb566b5eb7ced0f228c9854199cb1816ab83b..b5e13eead3b9d8af537ddae57617264eb968a9d8 100644 --- a/software/monitor/surykatka.ini.jinja2 +++ b/software/monitor/surykatka.ini.jinja2 @@ -10,7 +10,7 @@ NAMESERVER = {%- endfor %} {% endif %} URL = -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- if 'url' in slave %} {{ slave['url'] }} {%- endif -%} diff --git a/software/monitor/test/test.py b/software/monitor/test/test.py index e055e95555e59fd20d62efb886f2d4f162e9e7de..e552ec7397a24a98a249c88b2332b77575ad2c6f 100644 --- a/software/monitor/test/test.py +++ b/software/monitor/test/test.py @@ -65,7 +65,7 @@ class ServicesTestCase(SlapOSInstanceTestCase): self.assertIn(expected_process_name, process_names) -class MonitorTestMixin(object): +class MonitorTestMixin: monitor_setup_url_key = 'monitor-setup-url' def test_monitor_setup(self): @@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin): ) def updateSurykatkaDict(self): - class_list = self.surykatka_dict.keys() - for class_ in class_list: + for class_ in self.surykatka_dict: update_dict = {} update_dict['ini-file'] = os.path.join( self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,)) @@ -200,7 +199,7 @@ class EdgeSlaveMixin(MonitorTestMixin): set( glob.glob( os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))), - set([q['ini-file'] for q in self.surykatka_dict.values()]) + {q['ini-file'] for q in self.surykatka_dict.values()} ) for info_dict in self.surykatka_dict.values(): self.assertEqual( @@ -243,14 +242,11 @@ class EdgeSlaveMixin(MonitorTestMixin): for info_dict in self.surykatka_dict.values(): if os.path.exists(info_dict['json-file']): os.unlink(info_dict['json-file']) - env = os.environ.copy() - env.pop('PYTHONPATH', None) try: - subprocess.check_call(info_dict['status-json'], shell=True, env=env) + subprocess.check_call(info_dict['status-json']) except subprocess.CalledProcessError as e: self.fail('%s failed with code %s and message %s' % ( info_dict['status-json'], e.returncode, e.output)) - self.assertTrue(os.path.exists(info_dict['json-file'])) with open(info_dict['json-file']) as fh: status_json = json.load(fh) self.assertIn('bot_status', status_json) @@ -680,8 +676,8 @@ INTERVAL = 120 TIMEOUT = 4 SQLITE = %(db_file)s URL = - https://www.erp5.org/ - https://www.erp5.com/"""} + https://www.erp5.com/ + https://www.erp5.org/"""} } @classmethod diff --git a/software/slapos-sr-testing/software-py3.cfg b/software/slapos-sr-testing/software-py3.cfg index 5f442accb904882243bc389ce8e429ba3b9de3e2..fa8dce504d7863e06afb8ccf67b29d4a546b4ab8 100644 --- a/software/slapos-sr-testing/software-py3.cfg +++ b/software/slapos-sr-testing/software-py3.cfg @@ -13,3 +13,4 @@ eggs -= [template] extra = + ${slapos.test.monitor-setup:setup} diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index 7545420c9c077226d5b5cc37afcdaffaa92169b7..9a0168699e2ebb80f650f494b10e894dfb44487b 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -217,7 +217,6 @@ extra = ${slapos.test.erp5-setup:setup} ${slapos.test.htmlvalidatorserver-setup:setup} ${slapos.test.slapos-master-setup:setup} - ${slapos.test.monitor-setup:setup} ${slapos.test.plantuml-setup:setup} ${slapos.test.powerdns-setup:setup} ${slapos.test.proftpd-setup:setup} diff --git a/stack/slapos.cfg b/stack/slapos.cfg index 586c9995e748acf7a427175ee59086e5c96b3b75..f6541e1aca521a5545cfc368b71cd8198c4f5ce3 100644 --- a/stack/slapos.cfg +++ b/stack/slapos.cfg @@ -100,7 +100,7 @@ eggs = slapos.libnetworkcache [versions] -setuptools = 40.4.3 +setuptools = 44.0.0 # Use SlapOS patched zc.buildout zc.buildout = 2.7.1+slapos001 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)