Skip to content

S
slapos.toolbox
Commit ab39414c authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

monitor: reimplement monitor following the new promise design in slapgrid 

promise are run by slapgrid now, monitor will collect result and generate final files
monitor runpromise script can be used to test a promise (which is implemented on the new promise design)
remove all generated script and logrotate code from monitor.py, they will be added in buildout
parent 53bfe1fe
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 731 additions and 1019 deletions
setup.py
View file @ ab39414c
......@@ -52,6 +52,7 @@ setup(name=name,
'PyRSS2Gen',
'dnspython',
'requests',
'jsonschema',
] + additional_install_requires,
extras_require = {
'lampconfigure': ["mysqlclient"], #needed for MySQL Database access
......
slapos/monitor/collect.py
View file @ ab39414c
......@@ -69,7 +69,7 @@ class ResourceCollect:
assert os.path.exists(db_path)
if db_path.endswith("collector.db"):
db_path = db_path[:-len("collector.db")]
self.db = Database(db_path)
self.db = Database(db_path, create=False)
self.consumption_utils = ConsumptionReportBase(self.db)
def has_table(self, name):
......@@ -255,7 +255,18 @@ def main():
initMemoryDataFile(mem_file)
initIODataFile(io_file)
with open(status_file, "w") as status_file:
status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
status_file.write(json.dumps({
"cpu_time": 0,
"cpu_percent": 0,
"memory_rss": 0,
"memory_percent": 0,
"io_rw_counter": 0,
"date": "",
"total_process": 0,
"disk_used": 0,
"io_cycles_counter": 0,
"cpu_num_threads": 0
}))
with open(resource_file, "w") as resource_file:
resource_file.write('[]')
exit(1)
......
slapos/monitor/globalstate.py
View file @ ab39414c
This diff is collapsed.
slapos/monitor/monitor.py
View file @ ab39414c
This diff is collapsed.
slapos/monitor/runpromise.py
View file @ ab39414c
This diff is collapsed.
slapos/test/monitor/testbootstrap.py
View file @ ab39414c
......@@ -12,15 +12,18 @@ class MonitorBootstrapTest(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(self.base_dir, 'plugin'))
os.mkdir(os.path.join(self.base_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
os.mkdir(os.path.join(self.base_dir, 'public'))
os.mkdir(os.path.join(self.base_dir, 'private'))
os.mkdir(os.path.join(self.base_dir, 'cron.d'))
os.mkdir(os.path.join(self.base_dir, 'log'))
os.mkdir(os.path.join(self.base_dir, 'log-backup'))
os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
os.mkdir(os.path.join(self.base_dir, 'webdav'))
os.mkdir(os.path.join(self.base_dir, 'run'))
os.mkdir(os.path.join(self.base_dir, 'private/documents'))
self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
......@@ -28,13 +31,12 @@ class MonitorBootstrapTest(unittest.TestCase):
self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
self.monitor_config_dict = dict(
monitor_conf=self.monitor_config_file,
base_dir=self.base_dir,
root_title="Monitor ROOT",
title="Monitor",
url_list="",
base_url="https://monitor.test.com",
monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
promise_folder=os.path.join(self.base_dir, 'promise'),
promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
public_folder=os.path.join(self.base_dir, 'public'),
public_path_list="",
......@@ -45,18 +47,18 @@ class MonitorBootstrapTest(unittest.TestCase):
)
self.monitor_conf = """[monitor]
parameter-file-path = %(base_dir)s/knowledge0.cfg
promise-folder = %(base_dir)s/promise
service-pid-folder = %(base_dir)s/run
monitor-promise-folder = %(base_dir)s/monitor-promise
private-folder = %(base_dir)s/private
public-folder = %(base_dir)s/public
public-path-list = %(public_path_list)s
private-path-list = %(private_path_list)s
crond-folder = %(base_dir)s/cron.d
logrotate-folder = %(base_dir)s/logrotate.d
report-folder = %(base_dir)s/monitor-report
root-title = %(root_title)s
pid-file = %(base_dir)s/monitor.pid
log-folder = %(base_dir)s/log
log-backup-folder = %(base_dir)s/log-backup
document-folder = %(base_dir)s/private/documents
parameter-list =
raw monitor-user admin
file sample %(base_dir)s/param
......@@ -75,6 +77,12 @@ service-pid-folder = %(base_dir)s/run
promise-output-file = %(base_dir)s/monitor-bootstrap-status
promise-runner = %(promise_run_script)s
randomsleep = /bin/echo sleep
[promises]
output-folder = %(base_dir)s/public
legacy-promise-folder = %(base_dir)s/promise
promise-folder = %(base_dir)s/plugin
partition-folder = %(base_dir)s
"""
self.opml_outline = """<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(base_url)s/public/feed" xmlUrl="%(base_url)s/public/feed" url="%(base_url)s/share/private/" />"""
......@@ -87,21 +95,18 @@ randomsleep = /bin/echo sleep
with open(file_path, 'w') as cfg:
cfg.write(config)
def configPromises(self, amount):
def configPromiseList(self, amount):
promise_dir = os.path.join(self.base_dir, 'promise')
plugin_dir = os.path.join(self.base_dir, 'plugin')
promse_content = "/bin/bash echo something"
for index in range(1, amount+1):
promise_file = os.path.join(promise_dir, 'monitor_promise-%s' % index)
self.writeContent(promise_file, promse_content)
os.chmod(promise_file, 0755)
def configReports(self, amount):
promise_dir = os.path.join(self.base_dir, 'monitor-report')
promse_content = "/bin/bash echo something"
for index in range(1, amount+1):
promise_file = os.path.join(promise_dir, 'monitor_report-%s' % index)
promise_file = os.path.join(plugin_dir, 'monitor_promise-%s.py' % index)
self.writeContent(promise_file, promse_content)
os.chmod(promise_file, 0755)
os.chmod(promise_file, 0644)
def checkOPML(self, url_list):
opml_title = "<title>%(root_title)s</title>" % self.monitor_config_dict
......@@ -116,54 +121,6 @@ randomsleep = /bin/echo sleep
base_url=url)
self.assertTrue(opml_outline in opml_content)
def check_promises(self, sequential=False):
promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-promises')
self.assertTrue(os.path.exists(promise_cron))
with open(promise_cron) as cronf:
promise_command_list = cronf.read()
if not sequential:
promise_entry = '* * * * * /bin/echo sleep 60 && %(promise_run_script)s --pid_path "%(promise_runner_pid)s" --output "%(public_folder)s" --promise_folder "%(promise_folder)s" --timeout_file "None" --monitor_promise_folder "%(monitor_promise_folder)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
entry_line = promise_entry % self.monitor_config_dict
self.assertTrue(entry_line in promise_command_list,
"%s not in %s" %(entry_line, promise_command_list))
else:
promise_entry = '* * * * * /bin/echo sleep 60 &&%(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
promise_dir = os.path.join(self.base_dir, 'promise')
for filename in os.listdir(promise_dir):
promise_dict = dict(
promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
promise_output=os.path.join(self.base_dir, 'public', '%s.status.json' % filename),
promise_executable=os.path.join(promise_dir, filename),
promise_name=filename
)
promise_dict.update(self.monitor_config_dict)
entry_line = promise_entry % promise_dict
self.assertTrue(entry_line in promise_command_list)
def check_report(self):
promise_entry = '* * * * * %(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(data_dir)s" --instance_name "%(title)s" --hosting_name "%(root_title)s" --promise_type "report"'
promise_dir = os.path.join(self.base_dir, 'monitor-report')
data_dir = os.path.join(self.base_dir, 'private', 'documents')
promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-reports')
self.assertTrue(os.path.exists(promise_cron))
with open(promise_cron) as cronf:
promise_command_list = cronf.read()
for filename in os.listdir(promise_dir):
promise_dict = dict(
promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
promise_output=os.path.join(data_dir, '%s.report.json' % filename),
promise_executable=os.path.join(promise_dir, filename),
promise_name=filename,
data_dir=data_dir
)
promise_dict.update(self.monitor_config_dict)
entry_line = promise_entry % promise_dict
self.assertTrue(entry_line in promise_command_list)
def check_folder_equals(self, source, destination):
self.assertTrue(os.path.isdir(source))
......@@ -261,7 +218,7 @@ randomsleep = /bin/echo sleep
self.assertTrue(os.path.exists(os.path.join(self.base_dir, 'private/documents')))
def test_monitor_bootstrap_promises(self):
self.configPromises(3)
self.configPromiseList(3)
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
......@@ -273,38 +230,12 @@ randomsleep = /bin/echo sleep
self.checkOPML([self.monitor_config_dict['base_url']])
self.check_promises()
# Check update promises_list
self.configPromises(5) # Add two promises
self.configPromiseList(5) # Add two promises
os.unlink(os.path.join(self.base_dir, 'promise', 'monitor_promise-2')) # drop promise number 2
instance2 = Monitoring(self.monitor_config_file)
instance2.bootstrapMonitor()
self.assertTrue(os.path.exists(promise_file))
self.check_promises()
def test_monitor_bootstrap_report(self):
self.configReports(3)
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
instance = Monitoring(self.monitor_config_file)
instance.bootstrapMonitor()
promise_file = os.path.join(self.base_dir, 'monitor-bootstrap-status')
self.assertTrue(os.path.exists(promise_file))
self.checkOPML([self.monitor_config_dict['base_url']])
self.check_report()
# Check update promises_list
self.configReports(5) # Add two promises
os.unlink(os.path.join(self.base_dir, 'monitor-report', 'monitor_report-1')) # drop promise number 2
instance2 = Monitoring(self.monitor_config_file)
instance2.bootstrapMonitor()
self.assertTrue(os.path.exists(promise_file))
self.check_report()
def test_monitor_bootstrap_genconfig(self):
config_content = self.monitor_conf % self.monitor_config_dict
......
slapos/test/monitor/testglobalstate.py
View file @ ab39414c
......@@ -5,50 +5,56 @@ import shutil
import tempfile
import unittest
import json
import pkg_resources
from slapos.monitor import globalstate
from slapos.monitor.runpromise import RunPromise, parseArguments
from slapos.monitor.runpromise import MonitorPromiseLauncher, getArgumentParser
from slapos.monitor.monitor import Monitoring
from jsonschema import validate
class MonitorGlobalTest(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(self.base_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
self.etc_dir = os.path.join(self.base_dir, 'etc')
self.output_dir = os.path.join(self.base_dir, '.slapgrid/promise/result')
os.mkdir(self.etc_dir)
os.mkdir(os.path.join(self.etc_dir, 'plugin'))
os.mkdir(os.path.join(self.etc_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'public'))
os.mkdir(os.path.join(self.base_dir, 'private'))
os.mkdir(os.path.join(self.base_dir, 'cron.d'))
os.mkdir(os.path.join(self.base_dir, 'log'))
os.mkdir(os.path.join(self.base_dir, 'log-backup'))
os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
os.mkdir(os.path.join(self.base_dir, 'webdav'))
os.mkdir(os.path.join(self.base_dir, 'run'))
os.mkdir(os.path.join(self.base_dir, 'private/documents'))
self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
self.writeContent(os.path.join(self.base_dir, 'monitor-htpasswd'), '12345')
self.writeContent(os.path.join(self.base_dir, 'instance.cfg'), """[instance]
name = Monitor
root-name = Monitor ROOT
computer = COMP-1234
ipv4 = 10.0.151.118
ipv6 = 2001:34c:1254:df3:89::5df3
software-release = http://some.url.com/software.cfg
software-type = default
partition = slappart10""")
self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
self.public_dir = os.path.join(self.base_dir, 'public')
self.private_dir = os.path.join(self.base_dir, 'private')
monitor_schema_string = \
pkg_resources.resource_string(
'slapos.monitor',
'doc/monitor_instance.schema.json')
self.monitor_instance_schema = json.loads(monitor_schema_string)
self.monitor_config_dict = dict(
monitor_conf=self.monitor_config_file,
base_dir=self.base_dir,
root_title="Monitor ROOT",
title="Monitor",
url_list="",
url_list="https://sub.monitor.test.com https://sub2.monitor.test.com",
base_url="https://monitor.test.com",
monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
promise_folder=os.path.join(self.base_dir, 'promise'),
etc_dir=self.etc_dir,
promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
public_folder=os.path.join(self.base_dir, 'public'),
public_path_list="",
......@@ -59,18 +65,18 @@ partition = slappart10""")
)
self.monitor_conf = """[monitor]
parameter-file-path = %(base_dir)s/knowledge0.cfg
promise-folder = %(base_dir)s/promise
service-pid-folder = %(base_dir)s/run
monitor-promise-folder = %(base_dir)s/monitor-promise
private-folder = %(base_dir)s/private
public-folder = %(base_dir)s/public
public-path-list = %(public_path_list)s
private-path-list = %(private_path_list)s
crond-folder = %(base_dir)s/cron.d
logrotate-folder = %(base_dir)s/logrotate.d
report-folder = %(base_dir)s/monitor-report
root-title = %(root_title)s
pid-file = %(base_dir)s/monitor.pid
log-folder = %(base_dir)s/log
log-backup-folder = %(base_dir)s/log-backup
document-folder = %(base_dir)s/private/documents
parameter-list =
raw monitor-user admin
file sample %(base_dir)s/param
......@@ -89,6 +95,21 @@ service-pid-folder = %(base_dir)s/run
promise-output-file = %(base_dir)s/monitor-bootstrap-status
promise-runner = %(promise_run_script)s
randomsleep = /bin/echo sleep
[promises]
output-folder = %(base_dir)s/public
legacy-promise-folder = %(etc_dir)s/promise
promise-folder = %(etc_dir)s/plugin
partition-folder = %(base_dir)s
computer-id = COMP-1234
partition-cert =
partition-key =
partition-id = slappart0
ipv6 = 2001:34c:1254:df3:89::5df3
software-release = http://some.url.com/software.cfg
master-url = http://10.0.151.118:50000
software-type = default
ipv4 = 10.0.151.118
"""
def tearDown(self):
......@@ -109,7 +130,7 @@ randomsleep = /bin/echo sleep
echo "%(output)s"
exit %(code)s
""" % result_dict
promise_path = os.path.join(self.base_dir, 'promise', name)
promise_path = os.path.join(self.etc_dir, 'promise', name)
self.writeContent(promise_path, content)
os.chmod(promise_path, 0755)
return promise_path
......@@ -118,18 +139,13 @@ exit %(code)s
pid_path = os.path.join(self.base_dir, 'run', 'monitor-promise.pid')
promise_cmd = [
'--pid_path',
'%s' % pid_path, '--output', os.path.join(self.base_dir, 'public'),
'--promise_folder', os.path.join(self.base_dir, 'promise'),
'--monitor_promise_folder', os.path.join(self.base_dir, 'monitor-promise'),
'--promise_type', 'status',
'--monitor_url', 'https://monitor.test.com/share/private/',
'--history_folder', os.path.join(self.base_dir, 'public'),
'--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
arg_parser = parseArguments()
'--pid-path',
'%s' % pid_path, '-c', self.monitor_config_file]
arg_parser = getArgumentParser()
return arg_parser.parse_args(promise_cmd)
def test_monitor_instance_state(self):
self.maxDiff = None
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
......@@ -141,31 +157,27 @@ exit %(code)s
self.writePromise('promise_3', success=False)
self.writePromise('promise_4')
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_1.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_2.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_3.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_4.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_1.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_2.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_3.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json')))
os.symlink(self.output_dir, '%s/public/promise' % self.base_dir)
# generate instance state files
globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
globalstate.run(self.monitor_config_file)
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json')))
self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json')))
expected_result = """{
"status": "ERROR",
"_embedded": {
"instance": {
"partition": "slappart10",
"ipv6": "2001:34c:1254:df3:89::5df3",
"computer": "COMP-1234",
"ipv4": "10.0.151.118",
"software-release": "http://some.url.com/software.cfg",
"software-type": "default"
}
},
"partition_id": "slappart0",
"ipv6": "2001:34c:1254:df3:89::5df3",
"ipv4": "10.0.151.118",
"software_release": "http://some.url.com/software.cfg",
"software_type": "default",
"parameters": [{
"key": "",
"value": "admin",
......@@ -206,7 +218,11 @@ exit %(code)s
},
"private_url": {
"href": "https://monitor.test.com/share/private/"
}
},
"related_monitor": [
{"href": "https://sub.monitor.test.com/share/public"},
{"href": "https://sub2.monitor.test.com/share/public"}
]
},
"aggregate_reference": "COMP-1234",
"type": "global",
......@@ -222,15 +238,22 @@ exit %(code)s
# all promises are OK now
self.writePromise('promise_2', success=True)
self.writePromise('promise_3', success=True)
promise_runner.runPromise()
globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
# rerun right now
promise_runner.config.force = True
promise_runner.start()
globalstate.run(self.monitor_config_file)
expected_result_dict = json.loads(expected_result)
expected_result_dict["status"] = "OK"
expected_result_dict["state"] = {'error': 0, 'success': 4}
instance_result_dict = None
with open(os.path.join(self.private_dir, 'monitor.global.json')) as r:
result = json.loads(r.read().decode("utf-8"))
instance_result_dict = json.loads(r.read().decode("utf-8"))
result = instance_result_dict.copy()
result.pop("date")
self.assertEquals(result,
expected_result_dict)
# validate returned json result
validate(instance_result_dict, self.monitor_instance_schema)
slapos/test/monitor/testrunpromise.py
View file @ ab39414c
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7