Merge branch 'master' into slaprunner

Conflicts:
	CHANGES.txt
	setup.py

CHANGES.txt

-0.25 (unreleased)
+0.25 (2012-06-19)
 =================
 
- * No changes yet.
+ * slapmonitor is fully functional. [Mohamadou Mbengue]
 
 0.24 (2012-04-19)
 =================

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.25-dev'
+version = '0.26-dev'
 name = 'slapos.toolbox'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/monitor.py

+# -*- coding: utf-8 -*-
 ##############################################################################
 #
 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
@@ -27,15 +28,12 @@
 import datetime
 import os
 import time
-import resource
 from time import strftime
 import logging
 import sys
 from lxml import etree as ElementTree
-import platform
 import psutil
 from optparse import OptionParser
-import tempfile
 import sqlite3.dbapi2 as sqlite3
 
 #define global variable for log file
@@ -47,13 +45,13 @@ class MonitoringTool(object):
     pass
 
   def get_cpu_and_memory_usage(self, proc):
-    """Return CPU and Memory usage (percent) and 
+    """Return CPU and Memory usage (percent) and
     the specific moment used for the measure"""
     return [proc.get_cpu_percent(), sum(proc.get_cpu_times()), proc.get_memory_percent(), proc.get_memory_info()[0], datetime.datetime.now()]
 
 class GenerateXML(object):
   """Return a XML file upon request by reading from database"""
-  
+
   def __init__(self, element_tree, path_database, path_xml):
     self.element_tree = element_tree
     self.path_database = path_database
@@ -61,12 +59,12 @@ class GenerateXML(object):
 
   def dump_xml(self):
     """This func read data from database and through
-    _write_xml_output write result on xml file"""  
+    _write_xml_output write result on xml file"""
 
     consumption_infos = []
     #This list hold the consuption infos in the following order
     #[CPU % usage, CPU time usage (seconds), Memory % usage, Memory space usage (byte), date, time]
-    
+
     conn = sqlite3.connect(self.path_database)
     cursor = conn.cursor()
     cursor.execute("SELECT * FROM data")
@@ -92,19 +90,19 @@ class GenerateXML(object):
     total_cpu_time = 0.00
     previous = 0.00
     first_time = False
-    
+
     #The total time that the cpu spent to work on it
     #Start-end time and date
     start_time = consumption_infos[0][5]
     end_time = consumption_infos[-1][5]
     start_date = consumption_infos[0][4]
     end_date = consumption_infos[-1][4]
-    
+
     for item in consumption_infos:
       cpu_time.append(item[1])
       memory_percentage.append(item[2])
       memory_space.append(item[3])
-  
+
     #For all the samples, we calculate CPU consumption time
     for indice,element in enumerate(cpu_time):
       if indice == 0:
@@ -125,8 +123,8 @@ class GenerateXML(object):
       total_cpu_time = cpu_time[-1] - first_sample
     #Else, we add the last sample to the total CPU consumption time
     else:
-      total_cpu_time = total_cpu_time + cpu_time[-1] 
-    
+      total_cpu_time = total_cpu_time + cpu_time[-1]
+
     return [total_cpu_time, sum(memory_space) / float(len(memory_space)), start_time, end_time, start_date, end_date]
     #return [scipy.mean(cpu_percentage), cpu_time, scipy.mean(memory_percentage),
     #      scipy.mean(memory_space), start_time, end_time, start_date, end_date]
@@ -134,26 +132,26 @@ class GenerateXML(object):
 
 
   def _write_xml_output(self, res_list, storage_path):
-    """This function provide to dump on xml the consumption infos, 
+    """This function provide to dump on xml the consumption infos,
     the res_list contains the following informations:
 
-    [CPU mean %, CPU whole usage (seconds), Memory mean %, Memory mean space usage (byte), 
+    [CPU mean %, CPU whole usage (seconds), Memory mean %, Memory mean space usage (byte),
     start_time, end_time, start_date, end_date]"""
 
     #XXX- NOTE
 
-    """The res_list has been recently changed, now it contains  
+    """The res_list has been recently changed, now it contains
     [CPU whole usage (seconds), Memory mean space usage (byte)]"""
-    
+
 
     res_list = map(str, res_list)
 
     cpu_list = ['CPU Consumption',
                 'CPU consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
-                res_list[0], 
+                res_list[0],
                 ]
 
-    memory_list = ['Memory consumption', 
+    memory_list = ['Memory consumption',
                   'Memory consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
                   res_list[1],
                   ]
@@ -168,10 +166,10 @@ class GenerateXML(object):
     report = ElementTree.tostring(tree)
     fd = open(storage_path, 'w')
     fd.write("<?xml version='1.0' encoding='utf-8'?>%s" % report)
-    fd.close() 
-  
+    fd.close()
+
   def _add_movement(self, consumption, single_resource_list):
-    
+
     child_consumption = ElementTree.SubElement(consumption, "movement")
 
     child_movement = ElementTree.SubElement(child_consumption, "resource")
@@ -196,7 +194,7 @@ class GenerateXML(object):
 def parse_opt():
 
   usage="""usage: slapmonitor [options] PID_FILE_PATH DATABASE_PATH
-Usage: slapreport [options] XML_PATH DATABASE_PATH """
+Usage: slapreport [options] LOG_PATH DATABASE_PATH LOGBOX_IP LOGBOX_PORT LOGBOX_LOGIN LOGBOX_PASSWORD"""
 
   parser = OptionParser(usage=usage)
   parser.add_option('-t', '--update_time',type=int, dest='update_time', help='Specify the interval'\
@@ -206,39 +204,111 @@ Usage: slapreport [options] XML_PATH DATABASE_PATH """
   return parser
 
 class SlapMonitor(object):
-    
+
   def __init__(self, proc, update_time, path_database):
     self.proc = proc
     self.update_time = update_time
     self.path_database = path_database
-    self.start_monitor()        
-        
+    self.start_monitor()
+
 
   def start_monitor(self):
-        
+
     temporary_monitoring = MonitoringTool()
     #check if the process is alive == None, instead zero value == proc is over
     while self.proc.pid in psutil.get_pid_list():
       conn = sqlite3.connect(self.path_database)
       cursor = conn.cursor()
       cursor.execute("create table if not exists data (cpu real, cpu_time real, memory real, rss real," \
-                      "date text, time text)")
+                      "date text, time text, reported integer NULL DEFAULT 0)")
       try:
         res_list = temporary_monitoring.get_cpu_and_memory_usage(self.proc)
         date_catched = "-".join([str(res_list[4].year), str(res_list[4].month), str(res_list[4].day)])
         time_catched = ":".join([str(res_list[4].hour), str(res_list[4].minute), str(res_list[4].second)])
         res_list[4] = date_catched
         res_list.append(time_catched)
-        cursor.execute("insert into data values (?,?,?,?,?,?)" , res_list)
+        cursor.execute("insert into data(cpu, cpu_time, memory, rss, date, time) values (?,?,?,?,?,?)" , res_list)
         conn.commit()
         cursor.close()
         conn.close()
         time.sleep(self.update_time)
       except IOError:
-        if log_file: 
-          logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s" 
+        if log_file:
+          logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
                 % (self.proc.pid, strftime("%Y-%m-%d at %H:%m")))
-        sys.exit(1)        
+        sys.exit(1)
+    if log_file:
+      logging.info("EXIT 0: Process terminated normally!")
+    sys.exit(0)
+
+
+class SlapReport(object):
+  """
+  reports usage of apache and mariadb logs to an existing log server
+  """
+  def __init__(self, proc, update_time, consumption_log_path, database_path, ssh_parameters):
+    self.proc = proc
+    self.update_time = update_time
+    self.path_database = database_path
+    self.consumption_log_path = consumption_log_path
+    self.ssh_parameters = ssh_parameters
+    self.start_report()
+
+  def write_log(self):
+    """This func read data from database and through
+       write_log_file write result on a log file"""
+
+    #write none reported log in the consumption log log file
+    fd = open(self.consumption_log_path, 'a')
+    conn = sqlite3.connect(self.path_database)
+    cursor = conn.cursor()
+    log_list = cursor.execute("SELECT * FROM data WHERE reported=0 ORDER BY rowid ASC")
+    last_report_time = ""
+    for row in log_list:
+      cpu_consumption_info = "%s %s Instance %s CPU Consumption: CPU:%s CPU_TIME:%s\n" \
+          % (row[4],row[5], self.proc.name, row[0],row[1])
+      memory_consumption_info = "%s %s Instance %s Memory Consumption: %s\n" \
+          % (row[4], row[5], self.proc.name, row[2])
+      try:
+        cmd = "tail -n 2 %s | ssh %s:%s@%s -p %s " \
+            % (self.consumption_log_path, self.ssh_parameters['user'], \
+            self.ssh_parameters['passwd'], self.ssh_parameters['ip'], \
+            self.ssh_parameters['port'])
+        res = os.system(cmd)
+        if not res:
+          if last_report_time != row[5]:
+            fd.write("%s%s" % (cpu_consumption_info,memory_consumption_info))
+            last_report_time = "%s" % row[5]
+          cursor.execute("UPDATE data set reported='1' WHERE time=?", (row[5],))
+        conn.commit()
+
+      except Exception:
+        if log_file:
+          logging.info("ERROR : Unable to connect to % at %s"
+                       % (self.ssh_parameters['ip'], strftime("%Y-%m-%d at %H:%m")))
+          #sys.exit(1)
+
+    cursor.close()
+    conn.close()
+
+  def start_report(self):
+    """
+    while the process is running, connect to the database
+    and report the non-reported line,
+    when line is reported put 1 to column report
+    """
+    temporary_monitoring = MonitoringTool()
+    #check if the process is alive == None, instead zero value == proc is over
+    while self.proc.pid in psutil.get_pid_list():
+      try:
+        # send log to logbox
+        self.write_log()
+        time.sleep(self.update_time)
+      except IOError:
+        if log_file:
+          logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
+                       % (self.proc.pid, strftime("%Y-%m-%d at %H:%m")))
+          sys.exit(1)
     if log_file:
       logging.info("EXIT 0: Process terminated normally!")
     sys.exit(0)
@@ -252,17 +322,16 @@ def run_slapmonitor():
 
   if opts.path_log_file:
     logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
-    global log_file 
+    global log_file
     log_file = True
-  
+
   fed = open(args[0], 'r')
   pid_read = fed.read()
   fed.close()
   proc = psutil.Process(int(pid_read))
   SlapMonitor(proc, opts.update_time, args[1])
-  
 
-def run_slapreport():
+def run_slapreport_():
   #This function require the xml_path and database_path
   parser = parse_opt()
   opts, args = parser.parse_args()
@@ -277,3 +346,42 @@ def run_slapreport():
   get_xml_hand = GenerateXML(ElementTree, args[1], args[0])
   get_xml_hand.dump_xml()
 
+def run_slapreport():
+  #This function require the consumption_log_path and database_path ssh_parameters
+  parser = parse_opt()
+  ssh_parameters ={}
+  opts, args = parser.parse_args()
+
+  if len(args) != 7:
+    parser.error("Incorrect number of arguments, 7 required but "+str(len(args))+" detected" )
+
+  if opts.path_log_file:
+    logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
+    global log_file
+    log_file = True
+
+  if args[3] == "":
+    sys.exit(0)
+
+  pid_file_path = "%s" % args[0]
+  #set ssh parameters
+  ssh_parameters['ip']=args[3]
+  ssh_parameters['port']=args[4]
+  ssh_parameters['user']=args[5]
+  ssh_parameters['passwd']=args[6]
+
+  try:
+    fed = open(pid_file_path, 'r')
+    pid_read = fed.read()
+    fed.close()
+    proc = psutil.Process(int(pid_read))
+    SlapReport(proc, opts.update_time, args[1], args[2], ssh_parameters)
+  except IOError:
+    if log_file:
+      logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
+                   % (proc.pid, strftime("%Y-%m-%d at %H:%m")))
+      sys.exit(1)
+
+  if log_file:
+    logging.info("EXIT 0: Process terminated normally!")
+  #sys.exit(0)