Commit cd91ebea authored by Alain Takoudjou's avatar Alain Takoudjou

monitor: cleanup, improve performances by reducing size and amount of generated files

parent 21a25f0f
...@@ -26,12 +26,6 @@ download-only = true ...@@ -26,12 +26,6 @@ download-only = true
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
mode = 0644 mode = 0644
[monitor-web-base]
<= monitor-download-base
url = ${:_profile_base_location_}/web/${:filename}
destination = ${buildout:parts-directory}/monitor-web
on-update = true
[monitor-template-base] [monitor-template-base]
<= monitor-download-base <= monitor-download-base
url = ${:_profile_base_location_}/templates/${:filename} url = ${:_profile_base_location_}/templates/${:filename}
...@@ -61,14 +55,9 @@ eggs = ...@@ -61,14 +55,9 @@ eggs =
# Monitor templates files # Monitor templates files
[monitor-httpd-conf] [monitor-httpd-conf]
<= monitor-template-base <= monitor-template-base
md5sum = 65cf2c8f5cfffa509ec3fc06b37e7509 md5sum = 40dc51fc156f1ad7eb94be7f3cbf08b4
filename = monitor-httpd.conf.in filename = monitor-httpd.conf.in
[monitor-service-conf-template]
<= monitor-template-base
filename = monitor-service.cfg.in
md5sum = 5913d2a0096b50537f394a49b762b3e5
[monitor-template-wrapper] [monitor-template-wrapper]
<= monitor-template-base <= monitor-template-base
filename = wrapper.in filename = wrapper.in
...@@ -100,7 +89,7 @@ recipe = slapos.recipe.template:jinja2 ...@@ -100,7 +89,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg rendered = ${buildout:directory}/template-monitor.cfg
md5sum = 9d6f50e616655c1ae11ace419ac2861e md5sum = 7ac78495de73cadafea87f97428e487f
context = context =
key apache_location apache:location key apache_location apache:location
key gzip_location gzip:location key gzip_location gzip:location
...@@ -111,13 +100,11 @@ context = ...@@ -111,13 +100,11 @@ context =
raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename} raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename} raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename} raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
raw monitor_password_promise_template ${monitor-password-promise:location}/${monitor-password-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond raw dcron_executable_location ${dcron:location}/sbin/crond
raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename} raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
raw monitor_service_conf_template ${monitor-service-conf-template:location}/${monitor-service-conf-template:filename}
raw openssl_executable_location ${openssl:location}/bin/openssl raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable} raw python_executable ${buildout:executable}
raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter} raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
...@@ -131,31 +118,26 @@ depends = ...@@ -131,31 +118,26 @@ depends =
[monitor2-bin] [monitor2-bin]
<= monitor-template-script <= monitor-template-script
filename = monitor.py filename = monitor.py
md5sum = 280412f9024030c8cae343c2b07e5130 md5sum = 5525e7445dab16fd03f4eeccf069b74b
[run-promise-py] [run-promise-py]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/scripts/run-promise.py template = ${:_profile_base_location_}/scripts/run-promise.py
rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
md5sum = 641c6917bbd19a7abaebb120ccd4e713 md5sum = 97148dfbb730cc4f55ed54513ce823e0
mode = 0755 mode = 0755
context = context =
raw python ${buildout:directory}/bin/${extra-eggs:interpreter} raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
[monitor-password-promise]
<= monitor-template-script
filename = monitor-password-promise.py
md5sum = f7e937d6619eb674f39f34718928d91d
[status2rss-executable] [status2rss-executable]
<= monitor-template-script <= monitor-template-script
filename = status2rss.py filename = status2rss.py
md5sum = f297779d0881f4bd48081506efb492a4 md5sum = 88e3bf955e1e4eac76a444d50fa4f020
[monitor-globalstate] [monitor-globalstate]
<= monitor-template-script <= monitor-template-script
filename = globalstate.py filename = globalstate.py
md5sum = 1ebead4480d62d5e1d2612c2d9f4e1d8 md5sum = 3377e325baa4ecfcd6eee06945fb69fc
[monitor-collect] [monitor-collect]
<= monitor-template-script <= monitor-template-script
...@@ -165,12 +147,5 @@ md5sum = 78fbcb56761315bde354fe7914d3c54f ...@@ -165,12 +147,5 @@ md5sum = 78fbcb56761315bde354fe7914d3c54f
[monitor-document-edit] [monitor-document-edit]
<= monitor-template-script <= monitor-template-script
filename = monitor-document.py filename = monitor-document.py
md5sum = f3e557e5d81291a22d6d2837a9e37bd0 md5sum = 399ff4939b55ff74e6d48bec5a495981
[make-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/make-rss.sh.in
md5sum = 98c8f6fd81e405b0ad10db07c3776321
output = ${buildout:directory}/template-make-rss.sh.in
mode = 0644
...@@ -78,13 +78,8 @@ public = ${directory:monitor}/public ...@@ -78,13 +78,8 @@ public = ${directory:monitor}/public
private = ${directory:monitor}/private private = ${directory:monitor}/private
services = ${directory:services} services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d services-conf = ${directory:etc}/monitor.conf.d
www = ${directory:monitor}/web
web-dir = ${directory:monitor}/web
log = ${directory:log}/monitor log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor monitor-var = ${directory:var}/monitor
monitor-password-var = ${monitor-directory:monitor-var}/password
monitor-password-interface = ${monitor-directory:monitor-password-var}/password/interface
monitor-status2rss-var = ${monitor-directory:monitor-var}/status2rss
[logrotate-directory] [logrotate-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -135,7 +130,6 @@ public-folder = ${monitor-directory:public} ...@@ -135,7 +130,6 @@ public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private} private-folder = ${monitor-directory:private}
webdav-folder = ${monitor-directory:webdav} webdav-folder = ${monitor-directory:webdav}
report-folder = ${monitor-directory:reports} report-folder = ${monitor-directory:reports}
web-folder = ${monitor-directory:web-dir}
base-url = ${monitor-instance-parameter:monitor-base-url} base-url = ${monitor-instance-parameter:monitor-base-url}
monitor-hal-json = ${monitor-directory:public}/monitor.hal.json monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
service-pid-folder = ${monitor-directory:pids} service-pid-folder = ${monitor-directory:pids}
...@@ -230,7 +224,6 @@ password = ${monitor-instance-parameter:password} ...@@ -230,7 +224,6 @@ password = ${monitor-instance-parameter:password}
listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6} listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
port = ${monitor-instance-parameter:monitor-httpd-port} port = ${monitor-instance-parameter:monitor-httpd-port}
pid-file = ${directory:run}/monitor-httpd.pid pid-file = ${directory:run}/monitor-httpd.pid
cgid-pid-file = ${directory:run}/cgid.pid
access-log = ${monitor-directory:log}/monitor-httpd-access.log access-log = ${monitor-directory:log}/monitor-httpd-access.log
error-log = ${monitor-directory:log}/monitor-httpd-error.log error-log = ${monitor-directory:log}/monitor-httpd-error.log
cert-file = ${ca-directory:certs}/httpd.crt cert-file = ${ca-directory:certs}/httpd.crt
...@@ -279,7 +272,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file}) ...@@ -279,7 +272,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[monitor-status2rss-wrapper] [monitor-status2rss-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# XXX - hard-coded Urls # XXX - hard-coded Urls
command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}' command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}' --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
wrapper-path = ${directory:bin}/monitor-status2rss.py wrapper-path = ${directory:bin}/monitor-status2rss.py
...@@ -324,45 +317,6 @@ check-secure = 1 ...@@ -324,45 +317,6 @@ check-secure = 1
dash_path = {{ dash_executable_location }} dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }} curl_path = {{ curl_executable_location }}
[monitor-httpd-promise-conf]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:monitor-promise}/${monitor-httpd-promise:filename}.cfg
template = {{ monitor_service_conf_template }}
mode = 0644
context = section parameter_dict monitor-httpd-promise-conf-parameter
[monitor-httpd-promise-conf-parameter]
title = Monitor httpd listening
# frequency 5 minute hour day mounth weekday
frequency = */5 * * * *
public-path-list = ${monitor-httpd-conf-parameter:access-log} ${monitor-httpd-conf-parameter:error-log}
#private-path-list =
[monitor-password-parameter]
password-changed-once-path = ${directory:var}/monitor-password-changed-once
[monitor-password-promise]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_password_promise_template }}
rendered = ${directory:monitor-promise}/${:filename}
filename = monitor-password
mode = 0755
context =
raw python_executable {{ python_executable }}
key password_changed_once_path monitor-password-parameter:password-changed-once-path
[monitor-password-promise-conf-parameter]
title = Monitor password
frequency = */5 * * * *
private-path-list = ${monitor-directory:monitor-password-interface}
[monitor-password-promise-conf]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_service_conf_template }}
rendered = ${directory:monitor-promise}/${monitor-password-promise:filename}.cfg
mode = 0644
context = section parameter_dict monitor-password-promise-conf-parameter
[publish] [publish]
# XXX depends on monitor-base section # XXX depends on monitor-base section
monitor-base-url = ${monitor-base:base-url} monitor-base-url = ${monitor-base:base-url}
...@@ -376,6 +330,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random} ...@@ -376,6 +330,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
monitor-httpd-port = 8196 monitor-httpd-port = 8196
# XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port # XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
monitor-base-url = ${monitor-frontend-promise:url} monitor-base-url = ${monitor-frontend-promise:url}
# monitor-base-url = ${monitor-httpd-conf-parameter:url}
root-instance-title = ${slap-configuration:root-instance-title} root-instance-title = ${slap-configuration:root-instance-title}
monitor-url-list = monitor-url-list =
cors-domains = monitor.app.officejs.com cors-domains = monitor.app.officejs.com
...@@ -390,6 +345,8 @@ instance-configuration = ...@@ -390,6 +345,8 @@ instance-configuration =
configuration-file-path = ${monitor-directory:etc}/monitor_knowledge0.cfg configuration-file-path = ${monitor-directory:etc}/monitor_knowledge0.cfg
interface-url = https://monitor.app.officejs.com
[monitor-frontend] [monitor-frontend]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
...@@ -427,7 +384,6 @@ depends = ...@@ -427,7 +384,6 @@ depends =
${monitor-httpd-promise:filename} ${monitor-httpd-promise:filename}
${monitor-status2rss-cron-entry:name} ${monitor-status2rss-cron-entry:name}
#[buildout] [monitor-publish]
#parts = monitor-base-url = ${publish:monitor-base-url}
# monitor-base monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${publish:monitor-url}&username=$${publish:monitor-user}&password=${publish:monitor-password}
# publish
#!${dash-output:dash}
STATUS_DB={{ monitor_parameters['db-path'] }}
RSS_FILE={{ monitor_parameters['rss-path'] }}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${rss-bin:location}/${rss-bin:filename}
$PYTHON $STATUS2RSS "Monitoring RSS feed" "{{ monitor_parameters['url'] }}/{{ monitor_parameters['index-filename'] }}" $STATUS_DB > $RSS_FILE
...@@ -53,6 +53,8 @@ def main(args_list): ...@@ -53,6 +53,8 @@ def main(args_list):
statistic_folder = os.path.join(base_folder, 'data', '.jio_documents') statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json') parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if not os.path.exists(statistic_folder): if not os.path.exists(statistic_folder):
try: try:
os.makedirs(statistic_folder) os.makedirs(statistic_folder)
...@@ -66,7 +68,6 @@ def main(args_list): ...@@ -66,7 +68,6 @@ def main(args_list):
glob.glob("%s/*.status.json" % status_folder) glob.glob("%s/*.status.json" % status_folder)
) )
error = warning = success = 0 error = warning = success = 0
latest_date = ''
status = 'OK' status = 'OK'
promise_list = [] promise_list = []
global_state_file = os.path.join(base_folder, 'monitor.global.json') global_state_file = os.path.join(base_folder, 'monitor.global.json')
...@@ -84,18 +85,13 @@ def main(args_list): ...@@ -84,18 +85,13 @@ def main(args_list):
success += 1 success += 1
elif tmp_json['status'] == 'WARNING': elif tmp_json['status'] == 'WARNING':
warning += 1 warning += 1
if tmp_json['start-date'] > latest_date:
latest_date = tmp_json['start-date']
tmp_json['time'] = tmp_json['start-date'].split(' ')[1] tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
del tmp_json['start-date']
promise_list.append(tmp_json) promise_list.append(tmp_json)
if error: if error:
status = 'ERROR' status = 'ERROR'
elif warning: elif warning:
status = 'WARNING' status = 'WARNING'
if not latest_date:
latest_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
global_state_dict = dict( global_state_dict = dict(
status=status, status=status,
...@@ -105,7 +101,7 @@ def main(args_list): ...@@ -105,7 +101,7 @@ def main(args_list):
'warning': warning, 'warning': warning,
}, },
type='global', type='global',
date=latest_date, date=report_date,
_links={"rss_url": {"href": "%s/public/feed" % base_url}, _links={"rss_url": {"href": "%s/public/feed" % base_url},
"public_url": {"href": "%s/share/jio_public/" % base_url}, "public_url": {"href": "%s/share/jio_public/" % base_url},
"private_url": {"href": "%s/share/jio_private/" % base_url} "private_url": {"href": "%s/share/jio_private/" % base_url}
...@@ -150,7 +146,7 @@ def main(args_list): ...@@ -150,7 +146,7 @@ def main(args_list):
# Public information with the link to private folder # Public information with the link to private folder
public_state_dict = dict( public_state_dict = dict(
status=status, status=status,
date=latest_date, date=report_date,
_links={'monitor': {'href': '%s/share/jio_private/' % base_url}}, _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
title=global_state_dict.get('title', '') title=global_state_dict.get('title', '')
) )
......
...@@ -63,6 +63,9 @@ def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain): ...@@ -63,6 +63,9 @@ def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
try: try:
with open(old_httpd_cors_file, 'r') as cors_file: with open(old_httpd_cors_file, 'r') as cors_file:
if cors_file.read() == cors_domain: if cors_file.read() == cors_domain:
if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
# Skip if cors file is not empty
return True return True
except OSError, e: except OSError, e:
print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e)) print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
......
#!/usr/bin/env python
password_changed_once_path = "{{ password_changed_once_path }}"
import os
def main():
if os.path.exists(password_changed_once_path):
print('{"status":"OK"}')
return 0
print('{"status":"BAD","message":"Password never changed"}')
return 1
if __name__ == "__main__":
exit(main())
...@@ -8,10 +8,9 @@ import json ...@@ -8,10 +8,9 @@ import json
import ConfigParser import ConfigParser
import traceback import traceback
import argparse import argparse
import time
import glob
import urllib2 import urllib2
import ssl import ssl
import glob
from datetime import datetime from datetime import datetime
OPML_START = """<?xml version="1.0" encoding="UTF-8"?> OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
...@@ -39,24 +38,6 @@ def parseArguments(): ...@@ -39,24 +38,6 @@ def parseArguments():
parser.add_argument('--config_file', parser.add_argument('--config_file',
default='monitor.cfg', default='monitor.cfg',
help='Monitor Configuration file') help='Monitor Configuration file')
parser.add_argument('--promise-folder',
action='append', dest='promise_folder_list',
default=[],
help='The path to get promise executable files')
parser.add_argument('--public-folder',
action='append', dest='public_folder',
help='The path of public folder. All files in this folders will have public acess')
parser.add_argument('--private-folder',
action='append', dest='private_folder',
help='The path of private folder. All files in this folders will be accessible with password')
parser.add_argument('--promise-runner',
help='The path of promise runner, use to run promise files')
parser.add_argument('--wrapper-path',
help='Path of monitor generated promise scripts files.')
return parser.parse_args() return parser.parse_args()
...@@ -75,6 +56,13 @@ def softConfigGet(config, *args, **kwargs): ...@@ -75,6 +56,13 @@ def softConfigGet(config, *args, **kwargs):
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return None return None
def createSymlink(source, destination):
try:
os.symlink(source, destination)
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
class Monitoring(object): class Monitoring(object):
def __init__(self, configuration_file): def __init__(self, configuration_file):
...@@ -199,14 +187,6 @@ class Monitoring(object): ...@@ -199,14 +187,6 @@ class Monitoring(object):
self.promise_dict[filename] = {"path": path, self.promise_dict[filename] = {"path": path,
"configuration": ConfigParser.ConfigParser()} "configuration": ConfigParser.ConfigParser()}
# get promises configurations
#for filename in os.listdir(monitor_promise_folder):
# path = os.path.join(monitor_promise_folder, filename)
# if os.path.isfile(path) and filename[-4:] == ".cfg":
# promise_name = filename[:-4]
# if promise_name in promise_dict:
# loadConfig([path], promise_dict[promise_name]["configuration"])
def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""): def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
if destination_folder: if destination_folder:
if source_path_list: if source_path_list:
...@@ -277,32 +257,30 @@ class Monitoring(object): ...@@ -277,32 +257,30 @@ class Monitoring(object):
jio_private = os.path.join(self.webdav_folder, 'jio_private') jio_private = os.path.join(self.webdav_folder, 'jio_private')
mkdirAll(jio_public) mkdirAll(jio_public)
mkdirAll(jio_private) mkdirAll(jio_private)
try:
os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents')) createSymlink(self.public_folder,
except OSError, e: os.path.join(jio_public, '.jio_documents'))
if e.errno != os.errno.EEXIST: createSymlink(self.private_folder,
raise os.path.join(jio_private, '.jio_documents'))
try:
os.symlink(self.private_folder, os.path.join(jio_private, '.jio_documents'))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents') self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
self.document_folder = os.path.join(self.private_folder, 'documents')
config_folder = os.path.join(self.config_folder, '.jio_documents') config_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(self.data_folder) mkdirAll(self.data_folder)
mkdirAll(config_folder) mkdirAll(config_folder)
try:
os.symlink(os.path.join(self.private_folder, 'data'), createSymlink(os.path.join(self.private_folder, 'data'),
os.path.join(jio_private, 'data')) os.path.join(jio_private, 'data'))
except OSError, e: createSymlink(self.config_folder, os.path.join(jio_private, 'config'))
if e.errno != os.errno.EEXIST: createSymlink(self.data_folder, self.document_folder)
raise
# Cleanup private folder
for file in glob.glob("%s/*.history.json" % self.private_folder):
try: try:
os.symlink(self.config_folder, os.path.join(jio_private, 'config')) os.unlink(file)
except OSError, e: except OSError:
if e.errno != os.errno.EEXIST: print "failed to remove file %s. Ignoring..." % file
raise
def makeConfigurationFiles(self): def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents') config_folder = os.path.join(self.config_folder, '.jio_documents')
...@@ -352,12 +330,12 @@ class Monitoring(object): ...@@ -352,12 +330,12 @@ class Monitoring(object):
opml_content += OPML_OUTLINE_FEED % {'title': self.title, opml_content += OPML_OUTLINE_FEED % {'title': self.title,
'html_url': self.public_url + '/feed', 'html_url': self.public_url + '/feed',
'xml_url': self.public_url + '/feed', 'xml_url': self.public_url + '/feed',
'global_url': "%s/jio_public/" % self.webdav_url} 'global_url': "%s/jio_private/" % self.webdav_url}
for feed_url in feed_url_list: for feed_url in feed_url_list:
opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"), opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"),
'html_url': feed_url + '/public/feed', 'html_url': feed_url + '/public/feed',
'xml_url': feed_url + '/public/feed', 'xml_url': feed_url + '/public/feed',
'global_url': "%s/share/jio_public/" % feed_url} 'global_url': "%s/share/jio_private/" % feed_url}
opml_content += OPML_END opml_content += OPML_END
...@@ -376,37 +354,6 @@ class Monitoring(object): ...@@ -376,37 +354,6 @@ class Monitoring(object):
with open(file_path, 'w') as flog: with open(file_path, 'w') as flog:
flog.write(content) flog.write(content)
def generateMonitorHalJson(self):
monitor_link_dict = {"webdav": {"href": self.webdav_url},
"public": {"href": "%s/public" % self.webdav_url},
"private": {"href": "%s/private" % self.webdav_url},
"rss": {"href": "%s/feed" % self.public_url},
"jio_public": {"href": "%s/jio_public/" % self.webdav_url},
"jio_private": {"href": "%s/jio_private/" % self.webdav_url}
}
if self.title:
self.monitor_dict["title"] = self.title
if self.monitor_url_list:
monitor_link_dict["related_monitor"] = [{"href": url}
for url in self.monitor_url_list]
self.monitor_dict["_links"] = monitor_link_dict
if self.promise_items:
service_list = []
for service_name, promise in self.promise_items:
service_config = promise["configuration"]
tmp = softConfigGet(service_config, "service", "title")
service_dict = {}
service_dict["id"] = service_name
service_dict["_links"] = {"status": {"href": "%s/public/%s.status.json" % (self.webdav_url, service_name)}} # hardcoded
if tmp:
service_dict["title"] = tmp
service_list.append(service_dict)
self.monitor_dict["_embedded"] = {"service": service_list}
with open(self.monitor_hal_json, "w") as fp:
json.dump(self.monitor_dict, fp)
def generateReportCronEntries(self): def generateReportCronEntries(self):
cron_line_list = [] cron_line_list = []
...@@ -429,7 +376,7 @@ class Monitoring(object): ...@@ -429,7 +376,7 @@ class Monitoring(object):
'--promise_script "%s"' % report_script, '--promise_script "%s"' % report_script,
'--promise_name "%s"' % report_name, '--promise_name "%s"' % report_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded, '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.report_folder, '--history_folder "%s"' % self.data_folder,
'--instance_name "%s"' % self.title, '--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title, '--hosting_name "%s"' % self.root_title,
'--promise_type "report"'] '--promise_type "report"']
...@@ -484,11 +431,6 @@ class Monitoring(object): ...@@ -484,11 +431,6 @@ class Monitoring(object):
if service_name in service_name_list: if service_name in service_name_list:
service_name_list.pop(service_name_list.index(service_name)) service_name_list.pop(service_name_list.index(service_name))
"""wrapper_path = os.path.join(self.wraper_folder, service_name)
with open(wrapper_path, "w") as fp:
fp.write("#!/bin/sh\n%s" % command) # XXX hardcoded, use dash, sh or bash binary!
os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )"""
if service_name_list != []: if service_name_list != []:
# XXX Some service was removed, delete his status file so monitor will not consider his status anymore # XXX Some service was removed, delete his status file so monitor will not consider his status anymore
for service_name in service_name_list: for service_name in service_name_list:
...@@ -532,10 +474,6 @@ class Monitoring(object): ...@@ -532,10 +474,6 @@ class Monitoring(object):
private_path_list.split(), private_path_list.split(),
service_name) service_name)
# generate monitor.json
self.monitor_dict = {}
self.generateMonitorHalJson()
# Generate OPML file # Generate OPML file
self.generateOpmlFile(self.monitor_url_list, self.generateOpmlFile(self.monitor_url_list,
os.path.join(self.public_folder, 'feeds')) os.path.join(self.public_folder, 'feeds'))
...@@ -551,11 +489,13 @@ class Monitoring(object): ...@@ -551,11 +489,13 @@ class Monitoring(object):
# Rotate monitor data files # Rotate monitor data files
option_list = [ option_list = [
'daily', 'nocreate', 'noolddir', 'rotate 5', 'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
'nocompress', 'extension .json', 'dateext', 'nocompress', 'extension .json', 'dateext',
'dateformat -%Y-%m-%d', 'notifempty' 'dateformat -%Y-%m-%d', 'notifempty'
] ]
file_list = ["%s/*.data.json" % self.data_folder] file_list = [
"%s/*.data.json" % self.private_folder,
"%s/*.data.json" % self.data_folder]
self.generateLogrotateEntry('monitor.data', file_list, option_list) self.generateLogrotateEntry('monitor.data', file_list, option_list)
# Rotate public history status file, delete data of previous days # Rotate public history status file, delete data of previous days
......
#!/usr/bin/env python
import json
import os
import time
from datetime import datetime
OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
<!-- OPML generated by SlapOS -->
<opml version="1.1">
<head>
<title>SlapOS Monitoring Status Lists</title>
<dateCreated>%(creation_date)s</dateCreated>
<dateModified>%(mondification_date)s</dateModified>
</head>
<body>
<outline text="%(outline_title)s">"""
OPML_END = """ </outline>
</body>
</opml>"""
OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" />'
def main(config_file, output_file):
feed_url_list = []
if os.path.exists(output_file):
creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
else:
creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
with open(config_file, 'r') as fconfig:
feed_url_list = json.loads(fconfig.read())
opml_content = OPML_START
for feed_line in feed_url_list:
opml_content += OPML_OUTLINE_FEED % {'title': feed_line['title'], 'html_url': feed_line['url'], 'xml_url': feed_line['url']}
opml_content += OPML_END
with open(output_file, 'w') as wfile:
wfile.write(opml_content)
if __name__ == "__main__":
if len(sys.argv) < 3:
print("Usage: %s <rss_conf_file> <output_path>" % sys.argv[0])
sys.exit(2)
config_file = sys.argv[1]
output_file = sys.argv[2]
main(config_file, output_file)
\ No newline at end of file
...@@ -122,6 +122,12 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type): ...@@ -122,6 +122,12 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
} }
f_history.write(json.dumps(data_dict)) f_history.write(json.dumps(data_dict))
else: else:
# Remove useless informations
status_dict.pop('hosting_subscription', '')
status_dict.pop('title', '')
status_dict.pop('instance', '')
status_dict.pop('type', '')
with open (history_file, mode="r+") as f_history: with open (history_file, mode="r+") as f_history:
f_history.seek(0,2) f_history.seek(0,2)
position = f_history.tell() -2 position = f_history.tell() -2
......
import sys import sys
import os import os
import json import json
import datetime from datetime import datetime
import base64 import base64
import hashlib import hashlib
import PyRSS2Gen import PyRSS2Gen
...@@ -16,6 +16,8 @@ def parseArguments(): ...@@ -16,6 +16,8 @@ def parseArguments():
help='Path where to get *.status.json files which contain result of promises.') help='Path where to get *.status.json files which contain result of promises.')
parser.add_argument('--output', parser.add_argument('--output',
help='The Path of file where feed file will be saved.') help='The Path of file where feed file will be saved.')
parser.add_argument('--feed_url',
help='Url of this feed file.')
parser.add_argument('--public_url', parser.add_argument('--public_url',
help='Monitor Instance public URL.') help='Monitor Instance public URL.')
parser.add_argument('--private_url', parser.add_argument('--private_url',
...@@ -36,6 +38,7 @@ def main(): ...@@ -36,6 +38,7 @@ def main():
parser = parseArguments() parser = parseArguments()
rss_item_list = [] rss_item_list = []
report_date = datetime.utcnow()
for filename in os.listdir(parser.items_folder): for filename in os.listdir(parser.items_folder):
if filename.endswith(".status.json"): if filename.endswith(".status.json"):
filepath = os.path.join(parser.items_folder, filename) filepath = os.path.join(parser.items_folder, filename)
...@@ -46,7 +49,7 @@ def main(): ...@@ -46,7 +49,7 @@ def main():
print "Failed to load json file: %s" % filepath print "Failed to load json file: %s" % filepath
continue continue
description = result_dict.get('message', '') description = result_dict.get('message', '')
event_time = datetime.datetime.fromtimestamp(result_dict['change-time']) event_time = datetime.fromtimestamp(result_dict['change-time'])
rss_item = PyRSS2Gen.RSSItem( rss_item = PyRSS2Gen.RSSItem(
categories = [result_dict['status']], categories = [result_dict['status']],
source = PyRSS2Gen.Source(result_dict['title'], parser.public_url), source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
...@@ -55,7 +58,7 @@ def main(): ...@@ -55,7 +58,7 @@ def main():
description = "%s: %s\n%s" % (event_time, result_dict['status'], description), description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
link = parser.private_url, link = parser.private_url,
pubDate = event_time, pubDate = event_time,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, result_dict['status']))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
) )
rss_item_list.append(rss_item) rss_item_list.append(rss_item)
...@@ -64,11 +67,12 @@ def main(): ...@@ -64,11 +67,12 @@ def main():
sorted(rss_item_list, key=getKey) sorted(rss_item_list, key=getKey)
rss_feed = PyRSS2Gen.RSS2 ( rss_feed = PyRSS2Gen.RSS2 (
title = parser.instance_name, title = parser.instance_name,
link = parser.public_url, link = parser.feed_url,
description = parser.hosting_name, description = parser.hosting_name,
lastBuildDate = datetime.datetime.utcnow(), lastBuildDate = report_date,
items = rss_item_list items = rss_item_list
) )
with open(parser.output, 'w') as frss: with open(parser.output, 'w') as frss:
frss.write(rss_feed.to_xml()) frss.write(rss_feed.to_xml())
......
...@@ -11,7 +11,7 @@ ServerAdmin someone@email ...@@ -11,7 +11,7 @@ ServerAdmin someone@email
Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }} Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }}
Define MonitorPort Define MonitorPort
</IfDefine> </IfDefine>
DocumentRoot "{{ directory.get('webdav') }}" DocumentRoot "{{ directory.get('private') }}"
ErrorLog "{{ parameter_dict.get('error-log') }}" ErrorLog "{{ parameter_dict.get('error-log') }}"
LoadModule unixd_module modules/mod_unixd.so LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so LoadModule access_compat_module modules/mod_access_compat.so
...@@ -19,7 +19,6 @@ LoadModule authz_core_module modules/mod_authz_core.so ...@@ -19,7 +19,6 @@ LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authn_core_module modules/mod_authn_core.so LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so LoadModule authz_host_module modules/mod_authz_host.so
LoadModule mime_module modules/mod_mime.so LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule dir_module modules/mod_dir.so LoadModule dir_module modules/mod_dir.so
LoadModule ssl_module modules/mod_ssl.so LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so LoadModule alias_module modules/mod_alias.so
...@@ -55,31 +54,12 @@ SSLEngine On ...@@ -55,31 +54,12 @@ SSLEngine On
Include {{ parameter_dict.get('httpd-cors-config-file') }} Include {{ parameter_dict.get('httpd-cors-config-file') }}
Header set Vary Origin Header set Vary Origin
Header set Cache-Control "private, max-age=0, must-revalidate" Header set Cache-Control "private, max-age=40"
Header set Access-Control-Max-Age "0" Header set Access-Control-Max-Age "40"
Header unset ETag
Header set Pragma "no-cache"
Header set Access-Control-Allow-Credentials "true" Header set Access-Control-Allow-Credentials "true"
Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
{% if parameter_dict.has_key('monitor-url-list') -%}
RewriteEngine on
SSLProxyEngine on
ProxyPreserveHost On
SSLProxyVerify none
SSLProxyCheckPeerCN off
SSLProxyCheckPeerName off
{% set index=1 -%}
{% set monitor_url_list = parameter_dict.get('monitor-url-list').split('\n') -%}
{% for url in monitor_url_list -%}
{% if url.strip() -%}
RewriteRule /monitor{{ index }}/(.*) {{ url }}/$1 [L,P]
{% set index = index + 1 -%}
{% endif -%}
{% endfor -%}
{% endif -%}
DavLockDB {{ directory.get('monitor-var') }}/DavLock DavLockDB {{ directory.get('monitor-var') }}/DavLock
Alias /share {{ directory.get('webdav') }} Alias /share {{ directory.get('webdav') }}
<Directory {{ directory.get('webdav') }}> <Directory {{ directory.get('webdav') }}>
...@@ -101,22 +81,6 @@ Alias /share {{ directory.get('webdav') }} ...@@ -101,22 +81,6 @@ Alias /share {{ directory.get('webdav') }}
</Limit> </Limit>
</LocationMatch> </LocationMatch>
ScriptSock {{ parameter_dict.get('cgid-pid-file') }}
<Directory {{ directory.get('www') }}>
SSLVerifyDepth 1
SSLRequireSSL
SSLOptions +StrictRequire
# XXX: security????
DirectoryIndex index.html
Options FollowSymLinks
AllowOverride All
Order Deny,Allow
AuthType Basic
AuthName "Private access"
AuthUserFile "{{ parameter_dict.get('htpasswd-file') }}"
Require valid-user
</Directory>
Alias /private {{ directory.get('private') }}/ Alias /private {{ directory.get('private') }}/
<Directory {{ directory.get('private') }}> <Directory {{ directory.get('private') }}>
Order Deny,Allow Order Deny,Allow
...@@ -140,24 +104,6 @@ Alias /public {{ directory.get('public') }}/ ...@@ -140,24 +104,6 @@ Alias /public {{ directory.get('public') }}/
Allow from all Allow from all
</Directory> </Directory>
Alias /cgi-bin {{ directory.get('cgi-bin') }}
<Directory {{ directory.get('cgi-bin') }}>
# XXX security ???
Order Deny,Allow
Deny from all
<Files "*.cgi">
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Private access"
AuthUserFile "{{ parameter_dict.get('htpasswd-file') }}"
Require valid-user
</Files>
Options +ExecCGI
AddHandler cgi-script .cgi
Options Indexes FollowSymLinks
Satisfy all
</Directory>
{% if parameter_dict.get('httpd-include-file', '') -%} {% if parameter_dict.get('httpd-include-file', '') -%}
Include {{ parameter_dict.get('httpd-include-file') }} Include {{ parameter_dict.get('httpd-include-file') }}
{% endif -%} {% endif -%}
[service]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment