Commit cdd819f8 authored by Alain Takoudjou's avatar Alain Takoudjou

monitor: allow to generate rss from zope longrequest log

parent 011857f5
...@@ -139,7 +139,7 @@ mode = 0644 ...@@ -139,7 +139,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
download-only = true download-only = true
md5sum = 8add29b36060967209e54732967e8d80 md5sum = f880212b392a937a637f6eb244f0e6b5
filename = logTools.py filename = logTools.py
mode = 0644 mode = 0644
...@@ -147,7 +147,7 @@ mode = 0644 ...@@ -147,7 +147,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
download-only = true download-only = true
md5sum = 9617c1a06ef81c5141c8b8d08a985939 md5sum = 49eb40b5515db17bf9c63d87162ad4ad
filename = errorlog2rss.py.in filename = errorlog2rss.py.in
mode = 0644 mode = 0644
......
...@@ -16,7 +16,7 @@ base_link = "{{ base_link }}".strip() ...@@ -16,7 +16,7 @@ base_link = "{{ base_link }}".strip()
interval = {{ hour_interval }} interval = {{ hour_interval }}
rotate_time = {{ rotate_time }} rotate_time = {{ rotate_time }}
limit = {{ item_limit }} limit = {{ item_limit }}
PARSE_TYPE_LIST = ['zope'] PARSE_TYPE_LIST = ['zope', 'zope-longrequest']
if not os.path.exists(module_folder) or not os.path.isdir(module_folder): if not os.path.exists(module_folder) or not os.path.isdir(module_folder):
print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder
...@@ -33,6 +33,11 @@ if not type in PARSE_TYPE_LIST: ...@@ -33,6 +33,11 @@ if not type in PARSE_TYPE_LIST:
sys.path.append(module_folder) sys.path.append(module_folder)
import logTools import logTools
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
for name, log_path in log_file_list.items(): for name, log_path in log_file_list.items():
if not os.path.exists(log_path): if not os.path.exists(log_path):
print "WARNING: File %s not found..." % log_path print "WARNING: File %s not found..." % log_path
...@@ -40,16 +45,21 @@ for name, log_path in log_file_list.items(): ...@@ -40,16 +45,21 @@ for name, log_path in log_file_list.items():
rss_path = os.path.join(output_folder, name) rss_path = os.path.join(output_folder, name)
if not rss_path.endswith('.html'): if not rss_path.endswith('.html'):
rss_path += '.html' rss_path += '.html'
start_date = (datetime.now() -
timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
to_date = (datetime.now() -
timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
item_list = [] item_list = []
date_format = "%Y-%m-%d %H:%M:%S,%f"
log_filter = "ERROR"
if type == 'zope': if type == 'zope':
method = logTools.isZopeLogBeginLine method = logTools.isZopeLogBeginLine
zope_parser = logTools.getZopeParser() log_parser = logTools.getZopeParser()
item_list = logTools.parseLog(log_path, zope_parser, method, elif type == 'zope-longrequest':
filter_with="ERROR", start_date=start_date) log_filter = ""
method = logTools.isZopeLogBeginLine
log_parser = logTools.getZopeLogRequestParser()
item_list = logTools.parseLog(log_path, log_parser, method,
filter_with=log_filter, start_date=start_date,
date_format=date_format)
if item_list: if item_list:
start = 0 start = 0
size = len(item_list) size = len(item_list)
...@@ -60,10 +70,16 @@ for name, log_path in log_file_list.items(): ...@@ -60,10 +70,16 @@ for name, log_path in log_file_list.items():
logTools.insertRssDb(db_path, item_list[start:-1], name) logTools.insertRssDb(db_path, item_list[start:-1], name)
print "%s items inserted into database..." % limit print "%s items inserted into database..." % limit
logTools.truncateRssDb(db_path, to_date)
print "Generating RSS entries with %s items..." % limit print "Generating RSS entries with %s items..." % limit
logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit) logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
else:
print "No log entries with date upper than %s..." % start_date
try:
# Remove olders entries from database
logTools.truncateRssDb(db_path, to_date)
except Exception,e:
print "Database not exist!! ", str(e)
from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex, Literal
import os, re import os, re
import datetime import datetime
import uuid import uuid
...@@ -16,19 +16,35 @@ CREATE TABLE IF NOT EXISTS rss_entry ( ...@@ -16,19 +16,35 @@ CREATE TABLE IF NOT EXISTS rss_entry (
status VARCHAR(20), status VARCHAR(20),
method VARCHAR(25), method VARCHAR(25),
title VARCHAR(255), title VARCHAR(255),
content VARCHAR(255)); url VARCHAR(255),
content TEXT);
""") """)
db.commit() db.commit()
db.close() db.close()
def getZopeLogRequestParser():
integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
word = Word( alphas+nums+"@._-" )
info = Regex("([\d\w\s:\.]+;){2}")#Combine(word + ";" + Literal(" ") + word + ";")
request = Combine(Suppress("request: ") + Suppress(word+" ") + Regex(".*"))
no_request = Combine(Suppress("[No request]") + Regex(".*"))
bnf = serverDateTime.setResultsName("timestamp") + Suppress("-") + \
info.setResultsName("title") + \
(no_request | request).setResultsName("link")
return bnf
def getZopeParser(): def getZopeParser():
integer = Word(nums) integer = Word(nums)
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer) serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer)
status = Word(string.uppercase, max=7, min=3) status = Word(string.uppercase, max=7, min=3)
word = Word( alphas+nums+"@._-" ) word = Word( alphas+nums+"@._-" )
message = Regex(".*") message = Regex(".*")
bnf = serverDateTime.setResultsName("timestamp") + status.setResultsName("statusCode") + \ bnf = serverDateTime.setResultsName("timestamp") + \
word.setResultsName("method") + message.setResultsName("content") status.setResultsName("statusCode") + \
word.setResultsName("method") + message.setResultsName("title")
return bnf return bnf
def isZopeLogBeginLine(line): def isZopeLogBeginLine(line):
...@@ -68,8 +84,9 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f ...@@ -68,8 +84,9 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
fields.timestamp , date_format), fields.timestamp , date_format),
status=fields.get('statusCode', ''), status=fields.get('statusCode', ''),
method=fields.get('method', ''), method=fields.get('method', ''),
title=fields.content, url=fields.get('link', ''),
content=fields.content)) title=fields.title,
content=fields.get('content', fields.title)))
index += 1 index += 1
except Exception: except Exception:
raise raise
...@@ -81,8 +98,8 @@ def insertRssDb(db_path, entry_list, rss_name): ...@@ -81,8 +98,8 @@ def insertRssDb(db_path, entry_list, rss_name):
db = sqlite3.connect(db_path) db = sqlite3.connect(db_path)
for entry in entry_list: for entry in entry_list:
date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S') date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S')
db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)", db.execute("insert into rss_entry(name, datetime, status, method, title, url, content) values (?, ?, ?, ?, ?, ?, ?)",
(rss_name, date, entry['status'], entry['method'], entry['title'], entry['content'])) (rss_name, date, entry['status'], entry['method'], entry['title'], entry['url'], entry['content']))
db.commit() db.commit()
db.close() db.close()
...@@ -94,7 +111,7 @@ def truncateRssDb(db_path, to_date): ...@@ -94,7 +111,7 @@ def truncateRssDb(db_path, to_date):
def selectRssDb(db_path, rss_name, start_date, limit=0): def selectRssDb(db_path, rss_name, start_date, limit=0):
db = sqlite3.connect(db_path) db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry " query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC" query += "where name=? and datetime>=? order by datetime DESC"
if limit: if limit:
query += " limit ?" query += " limit ?"
...@@ -110,7 +127,7 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0): ...@@ -110,7 +127,7 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
items = [] items = []
db = sqlite3.connect(db_path) db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, content from rss_entry " query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC" query += "where name=? and datetime>=? order by datetime DESC"
if limit: if limit:
query += " limit ?" query += " limit ?"
...@@ -119,13 +136,14 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0): ...@@ -119,13 +136,14 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
entry_list = db.execute(query, (name, start_date)) entry_list = db.execute(query, (name, start_date))
for entry in entry_list: for entry in entry_list:
name, rss_date, status, method, title, content = entry name, rss_date, status, method, title, url, content = entry
if method: if method:
title = "[%s] %s" % (method, title) title = "[%s] %s" % (method, title)
if status:
title = "[%s] %s" % (status, title) title = "[%s] %s" % (status, title)
rss_item = PyRSS2Gen.RSSItem( rss_item = PyRSS2Gen.RSSItem(
title = title, title = title,
link = "", link = url,
description = content.replace('\n', '<br/>'), description = content.replace('\n', '<br/>'),
pubDate = rss_date, pubDate = rss_date,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link)))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment