Commit cdd819f8 authored by Alain Takoudjou's avatar Alain Takoudjou

monitor: allow to generate rss from zope longrequest log

parent 011857f5
......@@ -139,7 +139,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 8add29b36060967209e54732967e8d80
md5sum = f880212b392a937a637f6eb244f0e6b5
filename = logTools.py
mode = 0644
......@@ -147,7 +147,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 9617c1a06ef81c5141c8b8d08a985939
md5sum = 49eb40b5515db17bf9c63d87162ad4ad
filename = errorlog2rss.py.in
mode = 0644
......
......@@ -16,7 +16,7 @@ base_link = "{{ base_link }}".strip()
interval = {{ hour_interval }}
rotate_time = {{ rotate_time }}
limit = {{ item_limit }}
PARSE_TYPE_LIST = ['zope']
PARSE_TYPE_LIST = ['zope', 'zope-longrequest']
if not os.path.exists(module_folder) or not os.path.isdir(module_folder):
print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder
......@@ -33,6 +33,11 @@ if not type in PARSE_TYPE_LIST:
sys.path.append(module_folder)
import logTools
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
for name, log_path in log_file_list.items():
if not os.path.exists(log_path):
print "WARNING: File %s not found..." % log_path
......@@ -40,16 +45,21 @@ for name, log_path in log_file_list.items():
rss_path = os.path.join(output_folder, name)
if not rss_path.endswith('.html'):
rss_path += '.html'
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
item_list = []
date_format = "%Y-%m-%d %H:%M:%S,%f"
log_filter = "ERROR"
if type == 'zope':
method = logTools.isZopeLogBeginLine
zope_parser = logTools.getZopeParser()
item_list = logTools.parseLog(log_path, zope_parser, method,
filter_with="ERROR", start_date=start_date)
log_parser = logTools.getZopeParser()
elif type == 'zope-longrequest':
log_filter = ""
method = logTools.isZopeLogBeginLine
log_parser = logTools.getZopeLogRequestParser()
item_list = logTools.parseLog(log_path, log_parser, method,
filter_with=log_filter, start_date=start_date,
date_format=date_format)
if item_list:
start = 0
size = len(item_list)
......@@ -60,10 +70,16 @@ for name, log_path in log_file_list.items():
logTools.insertRssDb(db_path, item_list[start:-1], name)
print "%s items inserted into database..." % limit
logTools.truncateRssDb(db_path, to_date)
print "Generating RSS entries with %s items..." % limit
logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
else:
print "No log entries with date upper than %s..." % start_date
try:
# Remove olders entries from database
logTools.truncateRssDb(db_path, to_date)
except Exception,e:
print "Database not exist!! ", str(e)
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex, Literal
import os, re
import datetime
import uuid
......@@ -16,19 +16,35 @@ CREATE TABLE IF NOT EXISTS rss_entry (
status VARCHAR(20),
method VARCHAR(25),
title VARCHAR(255),
content VARCHAR(255));
url VARCHAR(255),
content TEXT);
""")
db.commit()
db.close()
def getZopeLogRequestParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
word = Word( alphas+nums+"@._-" )
info = Regex("([\d\w\s:\.]+;){2}")#Combine(word + ";" + Literal(" ") + word + ";")
request = Combine(Suppress("request: ") + Suppress(word+" ") + Regex(".*"))
no_request = Combine(Suppress("[No request]") + Regex(".*"))
bnf = serverDateTime.setResultsName("timestamp") + Suppress("-") + \
info.setResultsName("title") + \
(no_request | request).setResultsName("link")
return bnf
def getZopeParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
status = Word(string.uppercase, max=7, min=3)
word = Word( alphas+nums+"@._-" )
message = Regex(".*")
bnf = serverDateTime.setResultsName("timestamp") + status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("content")
bnf = serverDateTime.setResultsName("timestamp") + \
status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("title")
return bnf
def isZopeLogBeginLine(line):
......@@ -68,8 +84,9 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
fields.timestamp , date_format),
status=fields.get('statusCode', ''),
method=fields.get('method', ''),
title=fields.content,
content=fields.content))
url=fields.get('link', ''),
title=fields.title,
content=fields.get('content', fields.title)))
index += 1
except Exception:
raise
......@@ -81,8 +98,8 @@ def insertRssDb(db_path, entry_list, rss_name):
db = sqlite3.connect(db_path)
for entry in entry_list:
date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S')
db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['content']))
db.execute("insert into rss_entry(name, datetime, status, method, title, url, content) values (?, ?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['url'], entry['content']))
db.commit()
db.close()
......@@ -94,7 +111,7 @@ def truncateRssDb(db_path, to_date):
def selectRssDb(db_path, rss_name, start_date, limit=0):
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
......@@ -110,7 +127,7 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
items = []
db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry "
query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC"
if limit:
query += " limit ?"
......@@ -119,13 +136,14 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
entry_list = db.execute(query, (name, start_date))
for entry in entry_list:
name, rss_date, status, method, title, content = entry
name, rss_date, status, method, title, url, content = entry
if method:
title = "[%s] %s" % (method, title)
if status:
title = "[%s] %s" % (status, title)
rss_item = PyRSS2Gen.RSSItem(
title = title,
link = "",
link = url,
description = content.replace('\n', '<br/>'),
pubDate = rss_date,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment