monitor: make it work

stack/monitor/buildout.cfg

@@ -12,10 +12,20 @@ parts =
   dcron
   monitor-eggs
   extra-eggs
+  monitor-conf
   monitor-bin
+  monitor-web-index-html
+  monitor-web-monitor-css
+  monitor-web-monitor-js
   monitor-template
   rss-bin
 
+[monitor-download-base]
+recipe = hexagonit.recipe.download
+download-only = true
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
 [monitor-eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -36,25 +46,45 @@ md5sum = 98c8f6fd81e405b0ad10db07c3776321
 output = ${buildout:directory}/template-make-rss.sh.in
 mode = 0644
 
-[monitor-template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/monitor.cfg.in
-output = ${buildout:directory}/monitor.cfg
-filename = monitor.cfg
-md5sum = 51284c0aeb62eccd37f8a4e1621ee28c
-mode = 0644
+[monitor-conf]
+<= monitor-download-base
+filename = monitor.conf.in
+md5sum = 2db5c08c7e8658981b4b1e3f27fd5967
+
+[monitor-bin]
+<= monitor-download-base
+filename = monitor.py.in
+md5sum = 1a376c4063121db33ffab25b4cd99c76
+
+[monitor-web-index-html]
+<= monitor-download-base
+filename = index.html
+md5sum = 262db07691c145301252a49b6b51d11d
+
+[monitor-web-monitor-css]
+<= monitor-download-base
+filename = monitor.css
+md5sum = a18ab932e5e2e656995f47c7d4a7853a
 
-[monitor-site-template]
+[monitor-web-monitor-js]
+<= monitor-download-base
+filename = monitor.js.in
+md5sum = 4f8f1f7f26f589bfdae8fbfee74fc1cc
+
+[monitor-template]
 recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = c792bdee9049d7061e36c12e2343eb87
+md5sum = c9bcc845671f78bc3e4c544aa84313e3
 context =
     key apache_location apache:location
     key gzip_location gzip:location
-    key monitor_static_html monitor-html-static:location
     raw monitor_bin ${monitor-bin:location}/${monitor-bin:filename}
+    raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
+    raw monitor_web_index_html ${monitor-web-index-html:location}/${monitor-web-index-html:filename}
+    raw monitor_web_monitor_css ${monitor-web-monitor-css:location}/${monitor-web-monitor-css:filename}
+    raw monitor_web_monitor_js ${monitor-web-monitor-js:location}/${monitor-web-monitor-js:filename}
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
@@ -97,21 +127,10 @@ url = https://nexedi.erp5.net/monitor-5c66a4518b466d45cb31cb7e8a225cde20247589.t
 #md5sum
 strip-top-level-dir = true
 
-[monitor-bin]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-download-only = true
-md5sum = 68a17075eeeebabd01965be2ee25721d
-filename = monitor.py.in
-mode = 0644
-
 [run-promise-py]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-download-only = true
-md5sum = 0454e5e28b83da48e52770c977eade8b
+<= monitor-download-base
 filename = run-promise.py
-mode = 0644
+md5sum = 6db26ce13becf8a190e34c14cb8b6f9f
 
 [monitor-httpd-template]
 recipe = hexagonit.recipe.download

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -47,11 +47,11 @@ log = ${:var}/log
 scripts = ${:etc}/run
 services = ${:etc}/service
 promises = ${:etc}/promise
-ssl = ${:etc}/ssl
 monitor = ${:srv}/monitor
 
 [monitor-directory]
 recipe = slapos.cookbook:mkdirectory
+bin = ${directory:bin}
 etc = ${directory:etc}
 run = ${directory:monitor}/run
 #run = ${directory:scripts}
@@ -59,8 +59,10 @@ pids = ${directory:run}/monitor
 cgi-bin = ${directory:monitor}/cgi-bin
 public = ${directory:monitor}/public
 private = ${directory:monitor}/private
+services = ${directory:services}
 services-conf = ${directory:etc}/monitor.conf.d
 www = ${directory:monitor}/web
+web-dir = ${directory:monitor}/web
 log = ${directory:log}/monitor
 promises = ${directory:monitor}/promise-scripts
 
@@ -80,16 +82,17 @@ log = ${buildout:directory}/var/log
 
 [ca-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = ${directory:ssl}/requests/
-private = ${directory:ssl}/private/
-certs = ${directory:ssl}/certs/
-newcerts = ${directory:ssl}/newcerts/
-crl = ${directory:ssl}/crl/
+root = ${directory:srv}/ssl
+requests = ${:root}/requests
+private = ${:root}/private
+certs = ${:root}/certs
+newcerts = ${:root}/newcerts
+crl = ${:root}/crl
 
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = {{ openssl_executable_location }}
-ca-dir = ${monitor-directory:ca-dir}
+ca-dir = ${ca-directory:root}
 requests-directory = ${ca-directory:requests}
 wrapper = ${monitor-directory:services}/certificate_authority
 ca-private = ${ca-directory:private}
@@ -105,17 +108,14 @@ cert-file = ${monitor-httpd-conf:cert-file}
 executable = ${httpd-wrapper:wrapper-path}
 wrapper = ${directory:services}/monitor-httpd
 
-[monitor]
-recipe = slapos.cookbook:zero-knowledge.write
-filename = ${monitor-directory:etc}/monitor.conf
-
-title = ${slap-parameter:monitor-title}
+[monitor-conf-parameters]
+title = ${monitor-instance-parameter:monitor-title}
 service-executable-dir = ${monitor-directory:run}
 template-service-run = {{ monitor_service_run }}
 public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
-web-folder = ${monitor-static-web:web-dir}
-monitor-hal-json = ${monitor-static-web:web-dir}/monitor.haljson
+web-folder = ${monitor-directory:web-dir}
+monitor-hal-json = ${monitor-directory:web-dir}/monitor.haljson
 service-pid-folder = ${monitor-directory:pids}
 crond-folder = ${logrotate-directory:cron-entries}
 public-path-list = 
@@ -124,6 +124,12 @@ private-path-list =
 
 monitor-url-list = 
 
+[monitor-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_conf_template }}
+rendered = ${directory:etc}/${:filename}
+filename = monitor.conf
+context = section parameter_dict monitor-conf-parameters
 
 [httpd-monitor-htaccess]
 recipe = plone.recipe.command
@@ -134,8 +140,8 @@ user = admin
 password = admin
 
 [monitor-httpd-conf]
-listening-ip = ${slap-network-information:global-ipv6}
-port = 9206
+listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
+port = ${monitor-instance-parameter:monitor-httpd-port}
 pid-file = ${directory:run}/httpd.pid
 cgid-pid-file = ${directory:run}/cgid.pid
 access-log = ${monitor-directory:log}/httpd-access.log
@@ -143,7 +149,7 @@ error-log = ${monitor-directory:log}/httpd-error.log
 cert-file = ${ca-directory:certs}/httpd.crt
 key-file = ${ca-directory:certs}/httpd.key
 htaccess-file = ${httpd-monitor-htaccess:htaccess-path}
-url = https://[${slap-network-information:global-ipv6}]:${:port}/
+url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}/
 
 [monitor-httpd-conf]
 recipe = slapos.recipe.template:jinja2
@@ -152,7 +158,7 @@ rendered = ${monitor-directory:etc}/monitor-httpd.conf
 mode = 0744
 context =
   section directory monitor-directory
-  section monitor_parameters monitor
+  section monitor_parameters monitor-conf
   section monitor_httpd monitor-httpd-conf
 
 [httpd-wrapper]
@@ -173,13 +179,24 @@ context =
     key content :command
 command = kill -USR1 $(cat ${monitor-httpd-conf:pid-file})
 
-[monitor-static-web]
-recipe = plone.recipe.command
-web-dir = ${monitor-directory:www}/
-command = 
-  cp -ax {{monitor_static_html}}/* ${:web-dir}
-update-command = 
-stop-on-error = true
+[monitor-web-index-html]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_web_index_html }}
+rendered = ${monitor-directory:web-dir}/index.html
+context =
+
+[monitor-web-monitor-css]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_web_monitor_css }}
+rendered = ${monitor-directory:web-dir}/monitor.css
+context =
+
+[monitor-web-monitor-js]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_web_monitor_js }}
+rendered = ${monitor-directory:web-dir}/monitor.js
+context =
+  key monitor_title monitor-instance-parameter:monitor-title
 
 [start-monitor]
 recipe = slapos.recipe.template:jinja2
@@ -187,12 +204,14 @@ template = {{ monitor_bin }}
 rendered = ${directory:scripts}/bootstrap-monitor
 context = 
   raw python_executable {{ python_executable }}
-  key configuration_location monitor:filename
+  key public_folder monitor-directory:public
+  key configuration_location monitor-conf:rendered
+  key promise_runner_path monitor-run-promise:rendered
 
 [monitor-run-promise]
 recipe = slapos.recipe.template:jinja2
 template = {{ promise_executor_py }}
-rendered = ${directory:bin}/monitor-promise
+rendered = ${directory:bin}/monitor-run-promise
 mode = 700
 context = 
   raw python_executable {{ python_executable }}
@@ -240,19 +259,22 @@ title = Monitor promise httpd
 executable-file = ${monitor-monitor-promise:wrapper-path}
 frequency = */5 * * * *
 
-[publish-connection-information]
+[monitor-publish]
 recipe = slapos.cookbook:publish
 monitor_url_v6 = ${monitor-httpd-conf:url}
 
-[slap-parameter]
+[monitor-instance-parameter]
 monitor-title = Monitoring interface
 
 [buildout]
 parts =
+  monitor-web-index-html
+  monitor-web-monitor-css
+  monitor-web-monitor-js
   cron-entry-logrotate
   certificate-authority
-  monitor
+  monitor-conf
   start-monitor
   ca-httpd
-  conf-monitor-promise
-  publish-connection-information
+#  conf-monitor-promise
+  monitor-publish

stack/monitor/monitor.conf.in

+[monitor]
+{% for key, value in parameter_dict.items() -%}
+{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
+{% endfor -%}

stack/monitor/monitor.js → stack/monitor/monitor.js.in

@@ -2,6 +2,8 @@
 (function () {
   "use strict";
 
+  var monitor_title = '{{ dumps(monitor_title)[5:-1] }}';
+
   function loadJson(url) {
     /*global XMLHttpRequest */
     return new Promise(function (resolve, reject) {
@@ -55,10 +57,15 @@
     return [value];
   }
 
-  ///////////////////
-
+  function softGetPropertyAsList(object, path) {
+    try {
+      return forceList(getProperty(object, path));
+    } catch (ignored) {
+      return [];
+    }
+  }
 
-  var monitor_json_list = [];
+  ///////////////////
 
   function htmlToElementList(html) {
     /*global document */
@@ -80,7 +87,7 @@
   }
 
   function loadAndRenderMonitorSection(root, monitor_dict, monitor_url) {
-    var table, service_list = forceList(softGetProperty(monitor_dict, ["_embedded", "service"]));
+    var table, service_list = softGetPropertyAsList(monitor_dict, ["_embedded", "service"]);
     if (!service_list) {
       root.textContent = "";
       return;
@@ -116,10 +123,10 @@
   function loadAndRenderMonitorJson(root) {
     root.textContent = "Loading monitor section...";
     return loadJson("monitor.haljson").then(function (monitor_dict) {
-      monitor_json_list.push(monitor_dict);
+      //monitor_json_list.push(monitor_dict);
       root.innerHTML = "";
-      var loading = loadAndRenderMonitorSection(root, monitor_dict), related_monitor_list = forceList(softGetProperty(monitor_dict, ["_links", "related_monitor"]));
-      if (!related_monitor_list) { return loading; }
+      var loading = loadAndRenderMonitorSection(root, monitor_dict), related_monitor_list = softGetPropertyAsList(monitor_dict, ["_links", "related_monitor"]);
+      if (!related_monitor_list.length) { return loading; }
       return Promise.all([loading, Promise.all(related_monitor_list.map(function (link) {
         var div = htmlToElementList("<div>Loading monitor section...</div>")[0];
         root.appendChild(div);
@@ -130,7 +137,7 @@
         return loadJson(link.href).catch(function (reason) {
           div.textContent = (reason && (reason.name + ": " + reason.message));
         }).then(function (monitor_dict) {
-          monitor_json_list.push(monitor_dict);
+          //monitor_json_list.push(monitor_dict);
           div.remove();
           return loadAndRenderMonitorSection(root, monitor_dict, link.href);
         });
@@ -141,15 +148,17 @@
   function bootstrap(root) {
     var element_list = htmlToElementList([
       "<header><a href=\"\" class=\"as-button\">Refresh</a></header>",
-      "<h1>KVM Monitoring interface</h1>",
+      "<h1>" + monitor_title + "</h1>",
       "<h2>System health status</h2>",
       "<p>This interface allow to see the status of several features, it may show problems and sometimes provides a way to fix them.</p>",
       "<p>Red square means the feature has a problem, green square means it is ok.</p>",
       "<p>You can click on a feature below to get more precise information.</p>"
     ].join("\n")), div = document.createElement("div"), tmp;
     [].forEach.call(element_list, function (element) {
+      if (element.parentNode.parentNode) { return; }
       root.appendChild(element);
     });
+    document.title = monitor_title;
     root.appendChild(div);
     /*global alert */
     tmp = loadAndRenderMonitorJson(div);

stack/monitor/monitor.py.in

 #!{{ python_executable }}
 configuration_location = "{{ configuration_location }}"
+promise_runner_path = "{{ promise_runner_path }}"
+public_folder = "{{ public_folder }}"
 
 import sys
 import os
@@ -12,9 +14,6 @@ import ConfigParser
 def main():
   # initialisation
   config = loadConfig([configuration_location])
-  # create symlinks from monitor.conf
-  createSymlinksFromConfig((config, "monitor", "public-folder"), (config, "monitor", "public-path-list"))
-  createSymlinksFromConfig((config, "monitor", "private-folder"), (config, "monitor", "private-path-list"))
   # search for configurations in monitor.conf.d
   configuration_folder_location = configuration_location + ".d"
   service_config_list = [
@@ -22,7 +21,7 @@ def main():
     for filename in os.listdir(configuration_folder_location)
     if os.path.isfile(os.path.join(configuration_folder_location, filename))
   ]
-  # search for promises in ...?
+  # search for promises in ...? or let promises be on some service conf files?
   # XXX see old monitor.py.in
   # generate monitor.json
   monitor_dict = {}
@@ -43,38 +42,39 @@ def main():
       tmp = softConfigGet(service_config, "service", "title")
       if tmp:
         service_dict["title"] = tmp
-      tmp = softConfigGet(service_config, "service", "interface-url")
+      tmp = softConfigGet(service_config, "service", "interface-path")
       if tmp:
-        service_dict["_links"]["interface"] = {"href": tmp}
+        service_dict["_links"]["interface"] = {"href": "/%s.html" % service_name}  # XXX hardcoded
   with open(config.get("monitor", "monitor-hal-json"), "w") as fp:
     json.dump(monitor_dict, fp)
   # create symlinks from service configurations
   for service_config in service_config_list:
-    createSymlinksFromConfig((config, "monitor", "public-folder"), (service_config, "service", "public-path-list"))
-    createSymlinksFromConfig((config, "monitor", "private-folder"), (service_config, "service", "private-path-list"))
-  # run scripts according to frequency
-  # XXX
-  
-  for filename in os.listdir(configuration_folder_location):
-    config_file = os.path.join(configuration_folder_location, filename)
-    script_config = loadConfig(config_file)
-    base_name = script_config.get("service", "name")
-    pid_file = os.path.join(config.get("monitor", "service-pid-folder"),
-                                        "%s.pid" % base_name)
-    service_run_location = os.path.join(config.get("monitor", "service-executable-dir"),
-                                        base_name)
-    cron_run_location = os.path.join(config.get("monitor", "crond-folder"),
-                                      base_name)
-    mapping_dict = {"configuration_location": config_file,
-                    "process_pid_file": pid_file}
-    createServiceWrapper(mapping_dict, service_run_location,
-                        config.get("monitor", "template-service-run"))
-    addCronEntry(cron_run_location,
-                [service_run_location],
-                script_config.get("service", "frequency"))
+    service_name = service_config.get("service", "name")
+    createSymlinksFromConfig((config, "monitor", "public-folder"), (service_config, "service", "public-path-list"), service_name)
+    createSymlinksFromConfig((config, "monitor", "private-folder"), (service_config, "service", "private-path-list"), service_name)
+  # put promises to a cron file
+  # XXX if manifest is modified then:  # add manifest to avoid to write every minutes on cron.d...
+  service_pid_folder = config.get("monitor", "service-pid-folder")
+  crond_folder = config.get("monitor", "crond-folder")
+  cron_line_list = []
+  for service_config in service_config_list:
+    service_name = service_config.get("service", "name")
+    service_status_path = "%s/%s/status.json" % (public_folder, service_name)
+    mkdirAll(os.path.dirname(service_status_path))
+    service_cron_path = os.path.join(crond_folder, service_name)
+    cron_line_list.append("%s %s %s %s " % (
+      service_config.get("service", "frequency"),
+      promise_runner_path,
+      os.path.join(service_pid_folder, "%s.pid" % service_name),
+      service_status_path,
+    ) + service_config.get("service", "promise-path").replace("%", "\\%"))
+  with open(crond_folder + "/monitor-promises", "w") as fp:
+    fp.write("\n".join(cron_line_list))
+  # create symlinks from monitor.conf
+  createSymlinksFromConfig((config, "monitor", "public-folder"), (config, "monitor", "public-path-list"))
+  createSymlinksFromConfig((config, "monitor", "private-folder"), (config, "monitor", "private-path-list"))
   return 0
 
-
 def loadConfig(pathes):
   config = ConfigParser.ConfigParser()
   config.read(pathes)
@@ -86,66 +86,27 @@ def softConfigGet(config, *args, **kwargs):
   except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
     return None
 
-def createSymlinksFromConfig(destination_folder_config_tuple, source_list_config_tuple):
+def createSymlinksFromConfig(destination_folder_config_tuple, source_list_config_tuple, service_name=""):
   destination_folder = softConfigGet(*destination_folder_config_tuple)
   if destination_folder:
     source_path_str = softConfigGet(*source_list_config_tuple)
     if source_path_str:
       for path in source_path_str.split():
+        dirname = os.path.join(destination_folder, service_name)
         try:
-          os.symlink(path, os.path.join(destination_folder, os.path.basename(path)))
+          mkdirAll(dirname)  # could also raise OSError
+          os.symlink(path, os.path.join(dirname, os.path.basename(path)))
         except OSError, e:
           if e.errno != os.errno.EEXIST:
             raise
 
-def createServiceWrapper(mapping_dict, wrapper_file, template_wrapper):
-  with open(template_wrapper, 'r') as template:
-    with open(wrapper_file, 'w') as wrapper:
-      wrapper.write(template.read() % mapping_dict)
-  os.chmod(wrapper_file, 0700)
-
-def addCronEntry(file_path, command_list, frequency):
-  with open(file_path, 'w') as cfile:
-    cfile.write("%s %s" % (frequency, ' '.join(command_list)))
-
-class Popen(subprocess.Popen):
-  def set_timeout(self, timeout):
-    self.set_timeout = None # assert we're not called twice
-    event = threading.Event()
-    killed = [False] # we just need a mutable
-    def t():
-      # do not call wait() or poll() because they're not thread-safe
-      if not event.wait(timeout) and self.returncode is None:
-        # race condition if waitpid completes just before the signal sent ?
-        self.terminate()
-        killed[0] = True
-        if event.wait(5):
-          return
-        if self.returncode is None:
-          self.kill() # same race as for terminate ?
-    t = threading.Thread(target=t)
-    t.daemon = True
-    t.start()
-    def isKilled():
-      event.set()
-      t.join()
-      return killed[0]
-    return isKilled
-
-def executePath(path):
-  # XXX script_timeout could be passed as parameters
-  script_timeout = 3600 # in seconds
-  with open(os.devnull, 'r+') as f:
-    p = Popen(command, cwd=instance_path,
-              env=None if sys.platform == 'cygwin' else {},
-              stdin=f, stdout=f, stderr=subprocess.PIPE)
-    killed = p.set_timeout(script_timeout)
-    stderr = p.communicate()[1]
-    if killed():
-      return "Timed Out"
-    elif p.returncode:
-      return stderr.strip()
-  return None
+def mkdirAll(path):
+  try:
+    os.makedirs(path)
+  except OSError, e:
+    if e.errno == os.errno.EEXIST and os.path.isdir(path):
+      pass
+    else: raise
 
 if __name__ == "__main__":
   sys.exit(main())

stack/monitor/run-promise.py

@@ -56,6 +56,5 @@ def executeCommand(args):
     stderr=subprocess.PIPE
   )
 
-
 if __name__ == "__main__":
   sys.exit(main())