monitor: allow to generate rss from zope longrequest log

stack/monitor/buildout.cfg

@@ -139,7 +139,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 8add29b36060967209e54732967e8d80
+md5sum = f880212b392a937a637f6eb244f0e6b5
 filename = logTools.py
 mode = 0644
 
@@ -147,7 +147,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 9617c1a06ef81c5141c8b8d08a985939
+md5sum = 49eb40b5515db17bf9c63d87162ad4ad
 filename = errorlog2rss.py.in
 mode = 0644
 

stack/monitor/errorlog2rss.py.in

@@ -16,7 +16,7 @@ base_link = "{{ base_link }}".strip()
 interval = {{ hour_interval }}
 rotate_time = {{ rotate_time }}
 limit = {{ item_limit }}
-PARSE_TYPE_LIST = ['zope']
+PARSE_TYPE_LIST = ['zope', 'zope-longrequest']
 
 if not os.path.exists(module_folder) or not os.path.isdir(module_folder):
   print "ERROR: Could not load log parsing Module. %s is not a directory. Exiting..." % module_folder
@@ -33,6 +33,11 @@ if not type in PARSE_TYPE_LIST:
 sys.path.append(module_folder)
 import logTools
 
+start_date = (datetime.now() - 
+                  timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
+to_date = (datetime.now() - 
+                  timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
+
 for name, log_path in log_file_list.items():
   if not os.path.exists(log_path):
     print "WARNING: File %s not found..." % log_path
@@ -40,16 +45,21 @@ for name, log_path in log_file_list.items():
   rss_path = os.path.join(output_folder, name)
   if not rss_path.endswith('.html'):
     rss_path += '.html'
-  start_date = (datetime.now() - 
-                  timedelta(hours=interval)).strftime('%Y-%m-%d %H:%M:%S')
-  to_date = (datetime.now() - 
-                  timedelta(hours=rotate_time)).strftime('%Y-%m-%d %H:%M:%S')
+    
   item_list = []
+  date_format = "%Y-%m-%d %H:%M:%S,%f"
+  log_filter = "ERROR"
   if type == 'zope':
     method = logTools.isZopeLogBeginLine
-    zope_parser = logTools.getZopeParser()
-    item_list = logTools.parseLog(log_path, zope_parser, method,
-                            filter_with="ERROR", start_date=start_date)
+    log_parser = logTools.getZopeParser()
+  elif type == 'zope-longrequest':
+    log_filter = ""
+    method = logTools.isZopeLogBeginLine
+    log_parser = logTools.getZopeLogRequestParser()
+  
+  item_list = logTools.parseLog(log_path, log_parser, method,
+                        filter_with=log_filter, start_date=start_date,
+                                      date_format=date_format)
   if item_list:
     start = 0
     size = len(item_list)
@@ -60,10 +70,16 @@ for name, log_path in log_file_list.items():
     logTools.insertRssDb(db_path, item_list[start:-1], name)
     print "%s items inserted into database..." % limit
     
-    logTools.truncateRssDb(db_path, to_date)
-    
     print "Generating RSS entries with %s items..." % limit
     logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit)
+  else:
+    print "No log entries with date upper than %s..." % start_date
+  
+try:
+  # Remove olders entries from database
+  logTools.truncateRssDb(db_path, to_date)
+except Exception,e:
+  print "Database not exist!! ", str(e)
 
 
 

stack/monitor/logTools.py

-from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex
+from pyparsing import Word, alphas, Suppress, Combine, nums, string, Optional, Regex, Literal
 import os, re
 import datetime
 import uuid
@@ -16,19 +16,35 @@ CREATE TABLE IF NOT EXISTS rss_entry (
   status VARCHAR(20),
   method VARCHAR(25),
   title VARCHAR(255),
-  content VARCHAR(255));
+  url VARCHAR(255),
+  content TEXT);
 """)
   db.commit()
   db.close()
 
+def getZopeLogRequestParser():
+  integer = Word(nums)
+  serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + 
+                        integer + ":" + integer + ":" + integer + "," + integer)
+  word = Word( alphas+nums+"@._-" )
+  info = Regex("([\d\w\s:\.]+;){2}")#Combine(word + ";" + Literal(" ") + word + ";")
+  request = Combine(Suppress("request: ") + Suppress(word+" ") + Regex(".*"))
+  no_request = Combine(Suppress("[No request]") + Regex(".*"))
+  bnf = serverDateTime.setResultsName("timestamp") +  Suppress("-") + \
+          info.setResultsName("title") + \
+          (no_request | request).setResultsName("link")
+  return bnf
+
 def getZopeParser():
   integer = Word(nums)
-  serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + integer + ":" + integer + ":" + integer + "," + integer)
+  serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + 
+                      integer + ":" + integer + ":" + integer + "," + integer)
   status = Word(string.uppercase, max=7, min=3)
   word = Word( alphas+nums+"@._-" )
   message = Regex(".*")
-  bnf = serverDateTime.setResultsName("timestamp") +  status.setResultsName("statusCode") + \
-          word.setResultsName("method") + message.setResultsName("content")
+  bnf = serverDateTime.setResultsName("timestamp") + \
+          status.setResultsName("statusCode") + \
+          word.setResultsName("method") + message.setResultsName("title")
   return bnf
 
 def isZopeLogBeginLine(line):
@@ -68,8 +84,9 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
                             fields.timestamp , date_format),
                             status=fields.get('statusCode', ''),
                             method=fields.get('method', ''),
-                            title=fields.content,
-                            content=fields.content))
+                            url=fields.get('link', ''),
+                            title=fields.title,
+                            content=fields.get('content', fields.title)))
           index += 1
         except Exception:
           raise
@@ -81,8 +98,8 @@ def insertRssDb(db_path, entry_list, rss_name):
   db = sqlite3.connect(db_path)
   for entry in entry_list:
     date = entry['datetime'].strftime('%Y-%m-%d %H:%M:%S')
-    db.execute("insert into rss_entry(name, datetime, status, method, title, content) values (?, ?, ?, ?, ?, ?)",
-                (rss_name, date, entry['status'], entry['method'], entry['title'], entry['content']))
+    db.execute("insert into rss_entry(name, datetime, status, method, title, url, content) values (?, ?, ?, ?, ?, ?, ?)",
+                (rss_name, date, entry['status'], entry['method'], entry['title'], entry['url'], entry['content']))
   db.commit()
   db.close()
 
@@ -94,7 +111,7 @@ def truncateRssDb(db_path, to_date):
 
 def selectRssDb(db_path, rss_name, start_date, limit=0):
   db = sqlite3.connect(db_path)
-  query = "select name, datetime, status, method, title, content from rss_entry "
+  query = "select name, datetime, status, method, title, url, content from rss_entry "
   query += "where name=? and datetime>=? order by datetime DESC"
   if limit:
     query += " limit ?"
@@ -110,7 +127,7 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
   items = []
   
   db = sqlite3.connect(db_path)
-  query = "select name, datetime, status, method, title, content from rss_entry "
+  query = "select name, datetime, status, method, title, url, content from rss_entry "
   query += "where name=? and datetime>=? order by datetime DESC"
   if limit:
     query += " limit ?"
@@ -119,13 +136,14 @@ def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0):
     entry_list = db.execute(query, (name, start_date))
   
   for entry in entry_list:
-    name, rss_date, status, method, title, content = entry
+    name, rss_date, status, method, title, url, content = entry
     if method:
       title = "[%s] %s" % (method, title)
-    title = "[%s] %s" % (status, title)
+    if status:
+      title = "[%s] %s" % (status, title)
     rss_item = PyRSS2Gen.RSSItem(
         title = title,
-        link = "",
+        link = url,
         description = content.replace('\n', '<br/>'),
         pubDate = rss_date,
         guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (rss_date, url_link)))