Commit 21a34c69 authored by Alain Takoudjou's avatar Alain Takoudjou Committed by Tristan Cavelier

monitor: Allow to generate rss for errorlog

parent 7f1f23e3
......@@ -30,6 +30,7 @@ eggs =
PyRSS2Gen
Jinja2
APacheDEX
pyparsing
[make-rss-script]
recipe = slapos.recipe.template
......@@ -134,6 +135,22 @@ download-only = true
filename = run-apachedex.py.in
mode = 0644
[log-tools]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
#md5sum =
filename = logTools.py
mode = 0644
[errorlog-2rss]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
#md5sum =
filename = errorlog2rss.py.in
mode = 0644
[dcron-service]
recipe = slapos.recipe.template
url = ${template-dcron-service:output}
......
#!{{ python_executable }}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import sys
import os
import subprocess
from datetime import datetime, timedelta
type = "{{ type }}".strip()
log_file_list = {{ log_file_list }}
module_folder = "{{ log_tool_folder }}".strip()
output_folder = "{{ log_as_rss_ouptut }}".strip()
db_path = "{{ db_location }}".strip()
base_link = "{{ base_link }}".strip()
interval = {{ hour_interval }}
rotate_time = {{ rotate_time }}
limit = {{ item_limit }}
PARSE_TYPE_LIST = ['zope']
if not os.path.exists(module_folder) or not os.path.isdir(module_folder):
print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder
exit(1)
if not os.path.exists(output_folder) or not os.path.isdir(output_folder):
print "ERROR: %s is not a directory. Exiting..." % output_folder
exit(1)
if not type in PARSE_TYPE_LIST:
print "ERROR: Could not parse specified log file type. Exiting..." % output_folder
exit(1)
sys.path.append(module_folder)
import logTools
for name, log_path in log_file_list.items():
if not os.path.exists(log_path):
print "WARNING: File %s not found..." % log_path
continue
rss_path = os.path.join(output_folder, name)
if not rss_path.endswith('.html'):
rss_path += '.html'
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
item_list = []
if type == 'zope':
method = logTools.isZopeLogBeginLine
zope_parser = logTools.getZopeParser()
item_list = logTools.parseLog(log_path, zope_parser, method,
filter_with="ERROR", start_date=start_date)
if item_list:
start = 0
size = len(item_list)
if limit and size > limit:
start = size - limit - 1
print "Found %s log entries matching date upper than %s..." % (size, start_date)
logTools.insertRssDb(db_path, item_list[start:-1], name)
print "%s items inserted into database..." % limit
logTools.truncateRssDb(db_path, to_date)
print "Generating RSS entries with %s items..." % limit
logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex
import os, re
import datetime
import uuid
import base64
import sqlite3
import PyRSS2Gen
def init_db(db_path):
db = sqlite3.connect(db_path)
c = db.cursor()
c.executescript("""
CREATE TABLE IF NOT EXISTS rss_entry (
name VARCHAR(25),
datetime VARCHAR(15),
status VARCHAR(20),
method VARCHAR(25),
title VARCHAR(255),
content VARCHAR(255));
""")
db.commit()
db.close()
def getZopeParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer)
status = Word(string.uppercase, max=7, min=3)
word = Word( alphas+nums+"@._-" )
message = Regex(".*")
bnf = serverDateTime.setResultsName("timestamp") + status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("content")
return bnf
def isZopeLogBeginLine(line):
# This expression will check if line start with a date string
# XXX - if line match expression, then regex.group() return the date
if not line or line.strip() == "------":
return None
regex = re.match(r"(^\d{2,4}-\d{2}-\d{1,2}\s+\d{2}:\d{2}:\d{2}?[,\d]+)",
line)
return regex
def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_format=""):
if not os.path.exists(path):
print "ERROR: cannot get file: %s" % path
return []
log_result = []
if not date_format:
date_format = "%Y-%m-%d %H:%M:%S,%f"
with open(path, 'r') as logfile:
index = 0
for line in logfile:
regex = method(line)
if not regex:
if index == 0 or line.strip() == "------":
continue
# Add this line to log content
log_result[index - 1]['content'] += ("\n" + line)
else:
try:
fields = parserbnf.parseString(line)
if filter_with and not fields.statusCode == filter_with:
continue
if start_date and regex.group() < start_date:
continue
log_result.append(dict(datetime=datetime.datetime.strptime(
fields.timestamp , date_format),
status=fields.get('statusCode', ''),
method=fields.get('method', ''),
title=fields.content,
content=fields.content))
index += 1
except Exception:
raise
# print "WARNING: Could not parse log line. %s \n << %s >>" % (str(e), line)
return log_result
def insertRssDb(db_path, entry_list, rss_name):
init_db(db_path)
db = sqlite3.connect(db_path)
for entry in entry_list:
date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S')
db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['content']))
db.commit()
db.close()
def truncateRssDb(db_path, to_date):
db = sqlite3.connect(db_path)
db.execute("delete from rss_entry where datetime<?", (to_date,))
db.commit()
db.close()
def selectRssDb(db_path, rss_name, start_date, limit=0):
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
rows = db.execute(query, (rss_name, start_date, limit))
else:
rows = db.execute(query, (rss_name, start_date))
#db.close()
if rows:
return rows
return []
def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
items = []
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
entry_list = db.execute(query, (name, start_date, limit))
else:
entry_list = db.execute(query, (name, start_date))
for entry in entry_list:
name, rss_date, status, method, title, content = entry
if method:
title = "[%s] %s" % (method, title)
title = "[%s] %s" % (status, title)
rss_item = PyRSS2Gen.RSSItem(
title = title,
link = "",
description = content.replace('\n', '<br/>'),
pubDate = rss_date,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link)))
)
items.append(rss_item)
db.close()
### Build the rss feed
items.reverse()
rss_feed = PyRSS2Gen.RSS2 (
title = name,
link = url_link,
description = name,
lastBuildDate = datetime.datetime.utcnow(),
items = items
)
with open(rss_path, 'w') as rss_ouput:
rss_ouput.write(rss_feed.to_xml())
......@@ -19,6 +19,7 @@ index-filename = index.cgi
index-path = $${monitor-directory:www}/$${:index-filename}
db-path = $${monitor-directory:etc}/monitor.db
monitor-password-path = $${monitor-directory:etc}/.monitor.shadow
log-rss-directory = $${monitor-directory:log-rss-directory}
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
......@@ -53,6 +54,7 @@ monitor-result = $${:var}/monitor
apachedex-result = $${:srv}/apachedex
private-directory = $${:srv}/monitor-private
log-rss-directory = $${:srv}/ERRORLogAsRSS
[public-symlink]
recipe = cns.recipe.symlink
......@@ -189,7 +191,51 @@ context =
recipe = plone.recipe.command
command = ln -s $${:source} $${monitor-directory:private-directory}
source =
# ErrorLog URL As RSS
[error-log-rss-base]
recipe = slapos.recipe.template:jinja2
template = ${errorlog-2rss:location}/${errorlog-2rss:filename}
rendered = $${monitor-directory:bin}/$${:script-name}
mode = 0700
extensions = jinja2.ext.do
base-link = $${monitor-parameters:url}/private/
extra-context =
context =
raw python_executable ${buildout:directory}/bin/${extra-eggs:interpreter}
raw log_tool_folder ${log-tools:location}
raw db_location $${monitor-directory:etc}/$${:script-name}.db
key log_as_rss_ouptut monitor-directory:log-rss-directory
key base_link :base-link
$${:extra-context}
[log-as-rss-symlink]
< = monitor-directory-access
source = $${monitor-directory:log-rss-directory}
[error-log-rss]
<= error-log-rss-base
script-name = fileLog_AsRSS
# rotate_time: Max time (in hour) before remove an rss entry from database
# item_limit: Max number of rss entries to generate per cycle
extra-context =
section log_file_list error-log-list
raw item_limit 10
raw hour_interval 1
raw rotate_time 24
raw type zope
[error-log-list]
# XXX - error-log-name = FILE_PATH
[cron-entry-logAsRss]
<= cron
recipe = slapos.cookbook:cron.d
name = $${error-log-rss:script-name}
frequency = 30 * * * *
command = $${error-log-rss:rendered}
# Apache Dex
[apachedex-entries-base]
recipe = slapos.recipe.template:jinja2
template = ${run-apachedex:location}/${run-apachedex:filename}
......
......@@ -64,6 +64,13 @@ def init_db(db):
CREATE TABLE IF NOT EXISTS status (
timestamp INTEGER UNIQUE,
status VARCHAR(255));
CREATE TABLE IF NOT EXISTS rss_entry (
name VARCHAR(255),
timestamp INTEGER,
status VARCHAR(255),
method VARCHAR(255),
title VARCHAR(255),
content VARCHAR(255));
CREATE TABLE IF NOT EXISTS individual_status (
timestamp INTEGER,
status VARCHAR(255),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment