From 769ed5ec9550998a3b296dcfffa0aeb344a57e2e Mon Sep 17 00:00:00 2001 From: Bryton Lacquement <bryton.lacquement@nexedi.com> Date: Mon, 9 Mar 2020 17:07:37 +0100 Subject: [PATCH] monitor: target Python 3 --- component/surykatka/buildout.cfg | 21 ------------------- component/surykatka/buildout.hash.cfg | 17 --------------- component/surykatka/requirements.txt | 11 ---------- software/monitor/buildout.hash.cfg | 10 ++++----- .../instance-monitor-edgebot.cfg.jinja2 | 4 ++-- .../instance-monitor-edgetest.cfg.jinja2 | 8 +++---- software/monitor/instance.cfg | 9 ++------ software/monitor/script/collect_csv_dump.py | 3 ++- software/monitor/software.cfg | 20 +++++++++++++++++- software/monitor/surykatka.ini.jinja2 | 2 +- software/monitor/test/test.py | 11 ++++------ software/slapos-sr-testing/software-py3.cfg | 1 + software/slapos-sr-testing/software.cfg | 1 - 13 files changed, 40 insertions(+), 78 deletions(-) delete mode 100644 component/surykatka/buildout.cfg delete mode 100644 component/surykatka/buildout.hash.cfg delete mode 100644 component/surykatka/requirements.txt diff --git a/component/surykatka/buildout.cfg b/component/surykatka/buildout.cfg deleted file mode 100644 index d61c64c78..000000000 --- a/component/surykatka/buildout.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[buildout] -extends = - buildout.hash.cfg - ../python3/buildout.cfg - -parts = - surykatka - -[surykatka-requirements] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/${:_update_hash_filename_} - -[surykatka] -recipe = plone.recipe.command -command = - bash -c "${python3.7:executable} -m venv ${:location} && \ - . ${:location}/bin/activate && \ - pip install -r ${surykatka-requirements:target}" -location = ${buildout:parts-directory}/${:_buildout_section_name_} -stop-on-error = true -executable = ${:location}/bin/surykatka diff --git a/component/surykatka/buildout.hash.cfg b/component/surykatka/buildout.hash.cfg deleted file mode 100644 index 6dc5bac21..000000000 --- a/component/surykatka/buildout.hash.cfg +++ /dev/null @@ -1,17 +0,0 @@ -# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax. -# The only allowed lines here are (regexes): -# - "^#" comments, copied verbatim -# - "^[" section beginings, copied verbatim -# - lines containing an "=" sign which must fit in the following categorie. -# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file -# Copied verbatim. -# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported -# by the re-generation script. -# Re-generated. -# - other lines are copied verbatim -# Substitution (${...:...}), extension ([buildout] extends = ...) and -# section inheritance (< = ...) are NOT supported (but you should really -# not need these here). -[surykatka-requirements] -_update_hash_filename_ = requirements.txt -md5sum = c4bd35bcc5c9c25efe1edff5dd022605 diff --git a/component/surykatka/requirements.txt b/component/surykatka/requirements.txt deleted file mode 100644 index 4acc4a4f6..000000000 --- a/component/surykatka/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -Click==7.0 -certifi==2019.11.28 -chardet==3.0.4 -dnspython==1.16.0 -forcediphttpsadapter==1.0.1 -idna==2.8 -miniupnpc==2.0.2 -peewee==3.13.1 -requests==2.22.0 -surykatka==0.4.2 -urllib3==1.25.8 diff --git a/software/monitor/buildout.hash.cfg b/software/monitor/buildout.hash.cfg index a0167666a..e145e6228 100644 --- a/software/monitor/buildout.hash.cfg +++ b/software/monitor/buildout.hash.cfg @@ -14,7 +14,7 @@ # not need these here). [template] filename = instance.cfg -md5sum = dc9770bacea2c504b92ad2162e58d222 +md5sum = 07873511a15df74f0be761947819ac9b [template-monitor] _update_hash_filename_ = instance-monitor.cfg.jinja2 @@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 [template-monitor-edgetest] _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2 -md5sum = 9e237dbdda59e788202f0da194a57d41 +md5sum = a57106ee88ff3295b9ffce84105da79b [template-monitor-edgebot] _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 -md5sum = f28a329e830ed737d468abcb4e89e1a2 +md5sum = c535f4df6388cdc3f6c1df2a91d6dd53 [network-bench-cfg] filename = network_bench.cfg.in @@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 [monitor-collect-csv-dump] filename = script/collect_csv_dump.py -md5sum = cad2402bbd21907cfed6bc5af8c5d3ab +md5sum = 67a9d1c7173e0d99bf3a5c6eb15b2bd8 [template-surykatka-ini] _update_hash_filename_ = surykatka.ini.jinja2 -md5sum = a2de719a5a65438c8c3ee5195442beb6 +md5sum = 89545501f0e5bf11608978886429da3d diff --git a/software/monitor/instance-monitor-edgebot.cfg.jinja2 b/software/monitor/instance-monitor-edgebot.cfg.jinja2 index 1ba34633c..f499e419b 100644 --- a/software/monitor/instance-monitor-edgebot.cfg.jinja2 +++ b/software/monitor/instance-monitor-edgebot.cfg.jinja2 @@ -9,7 +9,7 @@ {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %} {%- if extra_slave_instance_list %} {#- Create slaves to process with setting up defaults #} -{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %} +{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %} {%- if 'check-status-code' not in slave %} {%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %} {%- endif %} @@ -37,7 +37,7 @@ {%- set part_list = [] %} {%- for class, slave_instance_list in slave_instance_dict.items() %} {#- class is used to separate surykatka with different timeouts #} -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %} {%- do part_list.append(part_id) %} {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %} diff --git a/software/monitor/instance-monitor-edgetest.cfg.jinja2 b/software/monitor/instance-monitor-edgetest.cfg.jinja2 index b3e147cd6..0b628af46 100644 --- a/software/monitor/instance-monitor-edgetest.cfg.jinja2 +++ b/software/monitor/instance-monitor-edgetest.cfg.jinja2 @@ -27,7 +27,7 @@ extra-context = {%- set namebase = "edgebot" %} {%- set authorized_slave_list = [] %} {%- set monitor_base_url_dict = {} -%} -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- do authorized_slave_list.append(slave) %} {%- endfor %} {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %} @@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url} software-type = {{edgebot_software_type}} return = monitor-base-url -{% for section, edgebot_request in request_dict.iteritems() %} +{% for section, edgebot_request in request_dict.items() %} [{{section}}] <= replicate name = {{ edgebot_request.get('name') }} @@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }} {%- set slave_configuration_dict = slapparameter_dict %} {%- do slave_configuration_dict.update(edgebot_request.get('config')) %} {%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %} -{%- for config_key, config_value in slave_configuration_dict.iteritems() %} +{%- for config_key, config_value in slave_configuration_dict.items() %} config-{{ config_key }} = {{ dumps(config_value) }} {% endfor -%} {%- if edgebot_request.get('sla') %} -{%- for parameter, value in edgebot_request.get('sla').iteritems() %} +{%- for parameter, value in edgebot_request.get('sla').items() %} sla-{{ parameter }} = {{ value }} {%- endfor %} {%- else %} diff --git a/software/monitor/instance.cfg b/software/monitor/instance.cfg index a11a2bb74..29f197453 100644 --- a/software/monitor/instance.cfg +++ b/software/monitor/instance.cfg @@ -44,17 +44,12 @@ context = import json_module json raw template_json_edgetest_test ${json-test-template:target} mode = 0644 -[surykatka-wrapped] -recipe = slapos.cookbook:wrapper -wrapper-path = $${buildout:bin-directory}/surykatka-wrapped -command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable} - [instance-base-edgebot] recipe = slapos.recipe.template:jinja2 template = ${template-monitor-edgebot:target} rendered = $${buildout:directory}/template-monitor-edgebot.cfg extensions = jinja2.ext.do -surykatka-binary = ${surykatka:executable} +surykatka-binary = ${buildout:bin-directory}/surykatka template-surykatka-ini = ${template-surykatka-ini:target} context = import json_module json @@ -64,7 +59,7 @@ context = import json_module json key slapparameter_dict slap-configuration:configuration key slap_software_type slap-configuration:slap-software-type raw software_type edgebot - key surykatka_binary surykatka-wrapped:wrapper-path + key surykatka_binary :surykatka-binary key template_surykatka_ini :template-surykatka-ini raw buildout_bin ${buildout:bin-directory} raw monitor_template_output ${monitor-template:output} diff --git a/software/monitor/script/collect_csv_dump.py b/software/monitor/script/collect_csv_dump.py index 085176a9b..3810ca054 100644 --- a/software/monitor/script/collect_csv_dump.py +++ b/software/monitor/script/collect_csv_dump.py @@ -27,6 +27,7 @@ # ############################################################################## +from __future__ import print_function import os import argparse import csv @@ -96,6 +97,6 @@ if __name__ == "__main__": if not os.path.exists(parser.collector_db): - print "Collector database not found..." + print("Collector database not found...") dump_table_into_csv(Database(parser.collector_db), parser.output_folder) diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index 270e70311..b88fb7d2e 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -4,9 +4,11 @@ extends = buildout.hash.cfg ../../component/pycurl/buildout.cfg ../../component/python-cryptography/buildout.cfg - ../../component/surykatka/buildout.cfg ../../stack/monitor/buildout.cfg ../../stack/slapos.cfg + ../../component/python3/buildout.cfg + +python = python3 parts = slapos-cookbook @@ -68,6 +70,7 @@ eggs += ${python-cryptography:egg} slapos.toolbox slapos.core + surykatka scripts = networkbench @@ -78,6 +81,7 @@ scripts = monitor.genstatus monitor.configwrite check-computer-memory + surykatka [monitor-eggs] eggs += @@ -88,3 +92,17 @@ eggs += [versions] slapos.recipe.template = 4.4 + +surykatka = 0.4.2 + +# For surykatka 0.4.2 +click = 7.0 +certifi = 2019.11.28 +chardet = 3.0.4 +dnspython = 1.16.0 +forcediphttpsadapter = 1.0.1 +idna = 2.8 +miniupnpc = 2.0.2 +peewee = 3.13.1 +requests = 2.22.0 +urllib3 = 1.25.8 diff --git a/software/monitor/surykatka.ini.jinja2 b/software/monitor/surykatka.ini.jinja2 index 788fb566b..b5e13eead 100644 --- a/software/monitor/surykatka.ini.jinja2 +++ b/software/monitor/surykatka.ini.jinja2 @@ -10,7 +10,7 @@ NAMESERVER = {%- endfor %} {% endif %} URL = -{%- for slave in sorted(slave_instance_list) %} +{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- if 'url' in slave %} {{ slave['url'] }} {%- endif -%} diff --git a/software/monitor/test/test.py b/software/monitor/test/test.py index e055e9555..4114e5bdd 100644 --- a/software/monitor/test/test.py +++ b/software/monitor/test/test.py @@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin): ) def updateSurykatkaDict(self): - class_list = self.surykatka_dict.keys() - for class_ in class_list: + for class_ in self.surykatka_dict: update_dict = {} update_dict['ini-file'] = os.path.join( self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,)) @@ -243,10 +242,8 @@ class EdgeSlaveMixin(MonitorTestMixin): for info_dict in self.surykatka_dict.values(): if os.path.exists(info_dict['json-file']): os.unlink(info_dict['json-file']) - env = os.environ.copy() - env.pop('PYTHONPATH', None) try: - subprocess.check_call(info_dict['status-json'], shell=True, env=env) + subprocess.check_call(info_dict['status-json']) except subprocess.CalledProcessError as e: self.fail('%s failed with code %s and message %s' % ( info_dict['status-json'], e.returncode, e.output)) @@ -680,8 +677,8 @@ INTERVAL = 120 TIMEOUT = 4 SQLITE = %(db_file)s URL = - https://www.erp5.org/ - https://www.erp5.com/"""} + https://www.erp5.com/ + https://www.erp5.org/"""} } @classmethod diff --git a/software/slapos-sr-testing/software-py3.cfg b/software/slapos-sr-testing/software-py3.cfg index 5f442accb..fa8dce504 100644 --- a/software/slapos-sr-testing/software-py3.cfg +++ b/software/slapos-sr-testing/software-py3.cfg @@ -13,3 +13,4 @@ eggs -= [template] extra = + ${slapos.test.monitor-setup:setup} diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index 7545420c9..9a0168699 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -217,7 +217,6 @@ extra = ${slapos.test.erp5-setup:setup} ${slapos.test.htmlvalidatorserver-setup:setup} ${slapos.test.slapos-master-setup:setup} - ${slapos.test.monitor-setup:setup} ${slapos.test.plantuml-setup:setup} ${slapos.test.powerdns-setup:setup} ${slapos.test.proftpd-setup:setup} -- 2.30.9