Skip to content

slapos
slapos
Commit 01d7f0be authored by Tristan Cavelier's avatar Tristan Cavelier
Browse files
Options

monitor: make it work

parent 5a60ad67
Hide whitespace changes
Inline Side-by-side
Showing with 158 additions and 144 deletions
stack/monitor/buildout.cfg
View file @ 01d7f0be
......@@ -12,10 +12,20 @@ parts =
dcron
monitor-eggs
extra-eggs
monitor-conf
monitor-bin
monitor-web-index-html
monitor-web-monitor-css
monitor-web-monitor-js
monitor-template
rss-bin
[monitor-download-base]
recipe = hexagonit.recipe.download
download-only = true
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[monitor-eggs]
recipe = zc.recipe.egg
eggs =
......@@ -36,25 +46,45 @@ md5sum = 98c8f6fd81e405b0ad10db07c3776321
output = ${buildout:directory}/template-make-rss.sh.in
mode = 0644
[monitor-template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/monitor.cfg.in
output = ${buildout:directory}/monitor.cfg
filename = monitor.cfg
md5sum = 51284c0aeb62eccd37f8a4e1621ee28c
mode = 0644
[monitor-conf]
<= monitor-download-base
filename = monitor.conf.in
md5sum = 2db5c08c7e8658981b4b1e3f27fd5967
[monitor-bin]
<= monitor-download-base
filename = monitor.py.in
md5sum = 1a376c4063121db33ffab25b4cd99c76
[monitor-web-index-html]
<= monitor-download-base
filename = index.html
md5sum = 262db07691c145301252a49b6b51d11d
[monitor-web-monitor-css]
<= monitor-download-base
filename = monitor.css
md5sum = a18ab932e5e2e656995f47c7d4a7853a
[monitor-site-template]
[monitor-web-monitor-js]
<= monitor-download-base
filename = monitor.js.in
md5sum = 4f8f1f7f26f589bfdae8fbfee74fc1cc
[monitor-template]
recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum = c792bdee9049d7061e36c12e2343eb87
md5sum = c9bcc845671f78bc3e4c544aa84313e3
context =
key apache_location apache:location
key gzip_location gzip:location
key monitor_static_html monitor-html-static:location
raw monitor_bin ${monitor-bin:location}/${monitor-bin:filename}
raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
raw monitor_web_index_html ${monitor-web-index-html:location}/${monitor-web-index-html:filename}
raw monitor_web_monitor_css ${monitor-web-monitor-css:location}/${monitor-web-monitor-css:filename}
raw monitor_web_monitor_js ${monitor-web-monitor-js:location}/${monitor-web-monitor-js:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond
......@@ -97,21 +127,10 @@ url = https://nexedi.erp5.net/monitor-5c66a4518b466d45cb31cb7e8a225cde20247589.t
#md5sum
strip-top-level-dir = true
[monitor-bin]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 68a17075eeeebabd01965be2ee25721d
filename = monitor.py.in
mode = 0644
[run-promise-py]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 0454e5e28b83da48e52770c977eade8b
<= monitor-download-base
filename = run-promise.py
mode = 0644
md5sum = 6db26ce13becf8a190e34c14cb8b6f9f
[monitor-httpd-template]
recipe = hexagonit.recipe.download
......
stack/monitor/instance-monitor.cfg.jinja2.in
View file @ 01d7f0be
......@@ -47,11 +47,11 @@ log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promises = ${:etc}/promise
ssl = ${:etc}/ssl
monitor = ${:srv}/monitor
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
bin = ${directory:bin}
etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
......@@ -59,8 +59,10 @@ pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
public = ${directory:monitor}/public
private = ${directory:monitor}/private
services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d
www = ${directory:monitor}/web
web-dir = ${directory:monitor}/web
log = ${directory:log}/monitor
promises = ${directory:monitor}/promise-scripts
......@@ -80,16 +82,17 @@ log = ${buildout:directory}/var/log
[ca-directory]
recipe = slapos.cookbook:mkdirectory
requests = ${directory:ssl}/requests/
private = ${directory:ssl}/private/
certs = ${directory:ssl}/certs/
newcerts = ${directory:ssl}/newcerts/
crl = ${directory:ssl}/crl/
root = ${directory:srv}/ssl
requests = ${:root}/requests
private = ${:root}/private
certs = ${:root}/certs
newcerts = ${:root}/newcerts
crl = ${:root}/crl
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_executable_location }}
ca-dir = ${monitor-directory:ca-dir}
ca-dir = ${ca-directory:root}
requests-directory = ${ca-directory:requests}
wrapper = ${monitor-directory:services}/certificate_authority
ca-private = ${ca-directory:private}
......@@ -105,17 +108,14 @@ cert-file = ${monitor-httpd-conf:cert-file}
executable = ${httpd-wrapper:wrapper-path}
wrapper = ${directory:services}/monitor-httpd
[monitor]
recipe = slapos.cookbook:zero-knowledge.write
filename = ${monitor-directory:etc}/monitor.conf
title = ${slap-parameter:monitor-title}
[monitor-conf-parameters]
title = ${monitor-instance-parameter:monitor-title}
service-executable-dir = ${monitor-directory:run}
template-service-run = {{ monitor_service_run }}
public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private}
web-folder = ${monitor-static-web:web-dir}
monitor-hal-json = ${monitor-static-web:web-dir}/monitor.haljson
web-folder = ${monitor-directory:web-dir}
monitor-hal-json = ${monitor-directory:web-dir}/monitor.haljson
service-pid-folder = ${monitor-directory:pids}
crond-folder = ${logrotate-directory:cron-entries}
public-path-list =
......@@ -124,6 +124,12 @@ private-path-list =
monitor-url-list =
[monitor-conf]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_conf_template }}
rendered = ${directory:etc}/${:filename}
filename = monitor.conf
context = section parameter_dict monitor-conf-parameters
[httpd-monitor-htaccess]
recipe = plone.recipe.command
......@@ -134,8 +140,8 @@ user = admin
password = admin
[monitor-httpd-conf]
listening-ip = ${slap-network-information:global-ipv6}
port = 9206
listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
port = ${monitor-instance-parameter:monitor-httpd-port}
pid-file = ${directory:run}/httpd.pid
cgid-pid-file = ${directory:run}/cgid.pid
access-log = ${monitor-directory:log}/httpd-access.log
......@@ -143,7 +149,7 @@ error-log = ${monitor-directory:log}/httpd-error.log
cert-file = ${ca-directory:certs}/httpd.crt
key-file = ${ca-directory:certs}/httpd.key
htaccess-file = ${httpd-monitor-htaccess:htaccess-path}
url = https://[${slap-network-information:global-ipv6}]:${:port}/
url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}/
[monitor-httpd-conf]
recipe = slapos.recipe.template:jinja2
......@@ -152,7 +158,7 @@ rendered = ${monitor-directory:etc}/monitor-httpd.conf
mode = 0744
context =
section directory monitor-directory
section monitor_parameters monitor
section monitor_parameters monitor-conf
section monitor_httpd monitor-httpd-conf
[httpd-wrapper]
......@@ -173,13 +179,24 @@ context =
key content :command
command = kill -USR1 $(cat ${monitor-httpd-conf:pid-file})
[monitor-static-web]
recipe = plone.recipe.command
web-dir = ${monitor-directory:www}/
command =
cp -ax {{monitor_static_html}}/* ${:web-dir}
update-command =
stop-on-error = true
[monitor-web-index-html]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_web_index_html }}
rendered = ${monitor-directory:web-dir}/index.html
context =
[monitor-web-monitor-css]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_web_monitor_css }}
rendered = ${monitor-directory:web-dir}/monitor.css
context =
[monitor-web-monitor-js]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_web_monitor_js }}
rendered = ${monitor-directory:web-dir}/monitor.js
context =
key monitor_title monitor-instance-parameter:monitor-title
[start-monitor]
recipe = slapos.recipe.template:jinja2
......@@ -187,12 +204,14 @@ template = {{ monitor_bin }}
rendered = ${directory:scripts}/bootstrap-monitor
context =
raw python_executable {{ python_executable }}
key configuration_location monitor:filename
key public_folder monitor-directory:public
key configuration_location monitor-conf:rendered
key promise_runner_path monitor-run-promise:rendered
[monitor-run-promise]
recipe = slapos.recipe.template:jinja2
template = {{ promise_executor_py }}
rendered = ${directory:bin}/monitor-promise
rendered = ${directory:bin}/monitor-run-promise
mode = 700
context =
raw python_executable {{ python_executable }}
......@@ -240,19 +259,22 @@ title = Monitor promise httpd
executable-file = ${monitor-monitor-promise:wrapper-path}
frequency = */5 * * * *
[publish-connection-information]
[monitor-publish]
recipe = slapos.cookbook:publish
monitor_url_v6 = ${monitor-httpd-conf:url}
[slap-parameter]
[monitor-instance-parameter]
monitor-title = Monitoring interface
[buildout]
parts =
monitor-web-index-html
monitor-web-monitor-css
monitor-web-monitor-js
cron-entry-logrotate
certificate-authority
monitor
monitor-conf
start-monitor
ca-httpd
conf-monitor-promise
publish-connection-information
# conf-monitor-promise
monitor-publish
stack/monitor/monitor.conf.in 0 → 100644
View file @ 01d7f0be
[monitor]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
stack/monitor/monitor.js → stack/monitor/monitor.js.in
View file @ 01d7f0be
......@@ -2,6 +2,8 @@
(function () {
"use strict";
var monitor_title = '{{ dumps(monitor_title)[5:-1] }}';
function loadJson(url) {
/*global XMLHttpRequest */
return new Promise(function (resolve, reject) {
......@@ -55,10 +57,15 @@
return [value];
}
///////////////////
function softGetPropertyAsList(object, path) {
try {
return forceList(getProperty(object, path));
} catch (ignored) {
return [];
}
}
var monitor_json_list = [];
///////////////////
function htmlToElementList(html) {
/*global document */
......@@ -80,7 +87,7 @@
}
function loadAndRenderMonitorSection(root, monitor_dict, monitor_url) {
var table, service_list = forceList(softGetProperty(monitor_dict, ["_embedded", "service"]));
var table, service_list = softGetPropertyAsList(monitor_dict, ["_embedded", "service"]);
if (!service_list) {
root.textContent = "";
return;
......@@ -116,10 +123,10 @@
function loadAndRenderMonitorJson(root) {
root.textContent = "Loading monitor section...";
return loadJson("monitor.haljson").then(function (monitor_dict) {
monitor_json_list.push(monitor_dict);
//monitor_json_list.push(monitor_dict);
root.innerHTML = "";
var loading = loadAndRenderMonitorSection(root, monitor_dict), related_monitor_list = forceList(softGetProperty(monitor_dict, ["_links", "related_monitor"]));
if (!related_monitor_list) { return loading; }
var loading = loadAndRenderMonitorSection(root, monitor_dict), related_monitor_list = softGetPropertyAsList(monitor_dict, ["_links", "related_monitor"]);
if (!related_monitor_list.length) { return loading; }
return Promise.all([loading, Promise.all(related_monitor_list.map(function (link) {
var div = htmlToElementList("<div>Loading monitor section...</div>")[0];
root.appendChild(div);
......@@ -130,7 +137,7 @@
return loadJson(link.href).catch(function (reason) {
div.textContent = (reason && (reason.name + ": " + reason.message));
}).then(function (monitor_dict) {
monitor_json_list.push(monitor_dict);
//monitor_json_list.push(monitor_dict);
div.remove();
return loadAndRenderMonitorSection(root, monitor_dict, link.href);
});
......@@ -141,15 +148,17 @@
function bootstrap(root) {
var element_list = htmlToElementList([
"<header><a href=\"\" class=\"as-button\">Refresh</a></header>",
"<h1>KVM Monitoring interface</h1>",
"<h1>" + monitor_title + "</h1>",
"<h2>System health status</h2>",
"<p>This interface allow to see the status of several features, it may show problems and sometimes provides a way to fix them.</p>",
"<p>Red square means the feature has a problem, green square means it is ok.</p>",
"<p>You can click on a feature below to get more precise information.</p>"
].join("\n")), div = document.createElement("div"), tmp;
[].forEach.call(element_list, function (element) {
if (element.parentNode.parentNode) { return; }
root.appendChild(element);
});
document.title = monitor_title;
root.appendChild(div);
/*global alert */
tmp = loadAndRenderMonitorJson(div);
......
stack/monitor/monitor.py.in
View file @ 01d7f0be
#!{{ python_executable }}
configuration_location = "{{ configuration_location }}"
promise_runner_path = "{{ promise_runner_path }}"
public_folder = "{{ public_folder }}"
import sys
import os
......@@ -12,9 +14,6 @@ import ConfigParser
def main():
# initialisation
config = loadConfig([configuration_location])
# create symlinks from monitor.conf
createSymlinksFromConfig((config, "monitor", "public-folder"), (config, "monitor", "public-path-list"))
createSymlinksFromConfig((config, "monitor", "private-folder"), (config, "monitor", "private-path-list"))
# search for configurations in monitor.conf.d
configuration_folder_location = configuration_location + ".d"
service_config_list = [
......@@ -22,7 +21,7 @@ def main():
for filename in os.listdir(configuration_folder_location)
if os.path.isfile(os.path.join(configuration_folder_location, filename))
]
# search for promises in ...?
# search for promises in ...? or let promises be on some service conf files?
# XXX see old monitor.py.in
# generate monitor.json
monitor_dict = {}
......@@ -43,38 +42,39 @@ def main():
tmp = softConfigGet(service_config, "service", "title")
if tmp:
service_dict["title"] = tmp
tmp = softConfigGet(service_config, "service", "interface-url")
tmp = softConfigGet(service_config, "service", "interface-path")
if tmp:
service_dict["_links"]["interface"] = {"href": tmp}
service_dict["_links"]["interface"] = {"href": "/%s.html" % service_name} # XXX hardcoded
with open(config.get("monitor", "monitor-hal-json"), "w") as fp:
json.dump(monitor_dict, fp)
# create symlinks from service configurations
for service_config in service_config_list:
createSymlinksFromConfig((config, "monitor", "public-folder"), (service_config, "service", "public-path-list"))
createSymlinksFromConfig((config, "monitor", "private-folder"), (service_config, "service", "private-path-list"))
# run scripts according to frequency
# XXX
for filename in os.listdir(configuration_folder_location):
config_file = os.path.join(configuration_folder_location, filename)
script_config = loadConfig(config_file)
base_name = script_config.get("service", "name")
pid_file = os.path.join(config.get("monitor", "service-pid-folder"),
"%s.pid" % base_name)
service_run_location = os.path.join(config.get("monitor", "service-executable-dir"),
base_name)
cron_run_location = os.path.join(config.get("monitor", "crond-folder"),
base_name)
mapping_dict = {"configuration_location": config_file,
"process_pid_file": pid_file}
createServiceWrapper(mapping_dict, service_run_location,
config.get("monitor", "template-service-run"))
addCronEntry(cron_run_location,
[service_run_location],
script_config.get("service", "frequency"))
service_name = service_config.get("service", "name")
createSymlinksFromConfig((config, "monitor", "public-folder"), (service_config, "service", "public-path-list"), service_name)
createSymlinksFromConfig((config, "monitor", "private-folder"), (service_config, "service", "private-path-list"), service_name)
# put promises to a cron file
# XXX if manifest is modified then: # add manifest to avoid to write every minutes on cron.d...
service_pid_folder = config.get("monitor", "service-pid-folder")
crond_folder = config.get("monitor", "crond-folder")
cron_line_list = []
for service_config in service_config_list:
service_name = service_config.get("service", "name")
service_status_path = "%s/%s/status.json" % (public_folder, service_name)
mkdirAll(os.path.dirname(service_status_path))
service_cron_path = os.path.join(crond_folder, service_name)
cron_line_list.append("%s %s %s %s " % (
service_config.get("service", "frequency"),
promise_runner_path,
os.path.join(service_pid_folder, "%s.pid" % service_name),
service_status_path,
) + service_config.get("service", "promise-path").replace("%", "\\%"))
with open(crond_folder + "/monitor-promises", "w") as fp:
fp.write("\n".join(cron_line_list))
# create symlinks from monitor.conf
createSymlinksFromConfig((config, "monitor", "public-folder"), (config, "monitor", "public-path-list"))
createSymlinksFromConfig((config, "monitor", "private-folder"), (config, "monitor", "private-path-list"))
return 0
def loadConfig(pathes):
config = ConfigParser.ConfigParser()
config.read(pathes)
......@@ -86,66 +86,27 @@ def softConfigGet(config, *args, **kwargs):
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return None
def createSymlinksFromConfig(destination_folder_config_tuple, source_list_config_tuple):
def createSymlinksFromConfig(destination_folder_config_tuple, source_list_config_tuple, service_name=""):
destination_folder = softConfigGet(*destination_folder_config_tuple)
if destination_folder:
source_path_str = softConfigGet(*source_list_config_tuple)
if source_path_str:
for path in source_path_str.split():
dirname = os.path.join(destination_folder, service_name)
try:
os.symlink(path, os.path.join(destination_folder, os.path.basename(path)))
mkdirAll(dirname) # could also raise OSError
os.symlink(path, os.path.join(dirname, os.path.basename(path)))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
def createServiceWrapper(mapping_dict, wrapper_file, template_wrapper):
with open(template_wrapper, 'r') as template:
with open(wrapper_file, 'w') as wrapper:
wrapper.write(template.read() % mapping_dict)
os.chmod(wrapper_file, 0700)
def addCronEntry(file_path, command_list, frequency):
with open(file_path, 'w') as cfile:
cfile.write("%s %s" % (frequency, ' '.join(command_list)))
class Popen(subprocess.Popen):
def set_timeout(self, timeout):
self.set_timeout = None # assert we're not called twice
event = threading.Event()
killed = [False] # we just need a mutable
def t():
# do not call wait() or poll() because they're not thread-safe
if not event.wait(timeout) and self.returncode is None:
# race condition if waitpid completes just before the signal sent ?
self.terminate()
killed[0] = True
if event.wait(5):
return
if self.returncode is None:
self.kill() # same race as for terminate ?
t = threading.Thread(target=t)
t.daemon = True
t.start()
def isKilled():
event.set()
t.join()
return killed[0]
return isKilled
def executePath(path):
# XXX script_timeout could be passed as parameters
script_timeout = 3600 # in seconds
with open(os.devnull, 'r+') as f:
p = Popen(command, cwd=instance_path,
env=None if sys.platform == 'cygwin' else {},
stdin=f, stdout=f, stderr=subprocess.PIPE)
killed = p.set_timeout(script_timeout)
stderr = p.communicate()[1]
if killed():
return "Timed Out"
elif p.returncode:
return stderr.strip()
return None
def mkdirAll(path):
try:
os.makedirs(path)
except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(path):
pass
else: raise
if __name__ == "__main__":
sys.exit(main())
stack/monitor/run-promise.py
View file @ 01d7f0be
......@@ -56,6 +56,5 @@ def executeCommand(args):
stderr=subprocess.PIPE
)
if __name__ == "__main__":
sys.exit(main())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7