monitor.py 14.6 KB
Newer Older
1
# -*- coding: utf-8 -*-
Łukasz Nowak's avatar
Łukasz Nowak committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
28

Łukasz Nowak's avatar
Łukasz Nowak committed
29 30
import datetime
import logging
31 32
import optparse
import os
Łukasz Nowak's avatar
Łukasz Nowak committed
33
import sys
34
import time
Łukasz Nowak's avatar
Łukasz Nowak committed
35
from lxml import etree as ElementTree
36

Łukasz Nowak's avatar
Łukasz Nowak committed
37
import sqlite3.dbapi2 as sqlite3
38
import psutil
Łukasz Nowak's avatar
Łukasz Nowak committed
39 40 41 42

#define global variable for log file
log_file = False

43 44 45 46 47 48 49

def read_pid(path):
  with open(path, 'r') as fin:
    # pid file might contain other stuff we don't care about (ie. postgres)
    pid = fin.readline()
    return int(pid)

Łukasz Nowak's avatar
Łukasz Nowak committed
50 51 52 53 54 55
class MonitoringTool(object):
  """Provide functions to monitor CPU and Memory"""
  def __init__(self):
    pass

  def get_cpu_and_memory_usage(self, proc):
56
    """Return CPU and Memory usage (percent) and
Łukasz Nowak's avatar
Łukasz Nowak committed
57 58 59 60 61
    the specific moment used for the measure"""
    return [proc.get_cpu_percent(), sum(proc.get_cpu_times()), proc.get_memory_percent(), proc.get_memory_info()[0], datetime.datetime.now()]

class GenerateXML(object):
  """Return a XML file upon request by reading from database"""
62

Łukasz Nowak's avatar
Łukasz Nowak committed
63 64 65 66 67 68 69
  def __init__(self, element_tree, path_database, path_xml):
    self.element_tree = element_tree
    self.path_database = path_database
    self.path_xml = path_xml

  def dump_xml(self):
    """This func read data from database and through
70
    _write_xml_output write result on xml file"""
Łukasz Nowak's avatar
Łukasz Nowak committed
71 72 73 74

    consumption_infos = []
    #This list hold the consuption infos in the following order
    #[CPU % usage, CPU time usage (seconds), Memory % usage, Memory space usage (byte), date, time]
75

Łukasz Nowak's avatar
Łukasz Nowak committed
76 77 78 79 80
    conn = sqlite3.connect(self.path_database)
    cursor = conn.cursor()
    cursor.execute("SELECT * FROM data")
    for row in cursor:
      consumption_infos.append(row)
81 82 83 84 85 86

    #If the database is not empty
    if consumption_infos != []:
      consumption_summary = self._eval_consumption_summary (consumption_infos)
      self._write_xml_output(consumption_summary, self.path_xml)

Łukasz Nowak's avatar
Łukasz Nowak committed
87 88 89 90 91
    #Once we got infos we delete the table 'data'
    cursor.execute("DELETE FROM data")
    conn.commit()
    cursor.close()
    conn.close()
92

Łukasz Nowak's avatar
Łukasz Nowak committed
93 94 95 96
  def _eval_consumption_summary(self, consumption_infos):
    """This function return a resources usage summary, for pricing purpose"""
    memory_percentage = []
    memory_space = []
97 98 99 100
    cpu_time = []
    total_cpu_time = 0.00
    previous = 0.00
    first_time = False
101

Łukasz Nowak's avatar
Łukasz Nowak committed
102
    #The total time that the cpu spent to work on it
103
    #Start-end time and date
Łukasz Nowak's avatar
Łukasz Nowak committed
104 105 106 107
    start_time = consumption_infos[0][5]
    end_time = consumption_infos[-1][5]
    start_date = consumption_infos[0][4]
    end_date = consumption_infos[-1][4]
108

Łukasz Nowak's avatar
Łukasz Nowak committed
109
    for item in consumption_infos:
110
      cpu_time.append(item[1])
Łukasz Nowak's avatar
Łukasz Nowak committed
111 112
      memory_percentage.append(item[2])
      memory_space.append(item[3])
113

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
    #For all the samples, we calculate CPU consumption time
    for indice,element in enumerate(cpu_time):
      if indice == 0:
        first_sample = float(element)
        first_time = True
      else:
        #If the partition has been restarted...
        if previous > float(element):
          #If the partition has been restarted for the first time...
          if first_time:
            #We count the usage before the stop
            previous = previous - first_sample
            first_time = False
          total_cpu_time = total_cpu_time + previous
        previous = float(element)
    #If the partition hasn't been restarted, we count only the difference between the last and the first sample
    if first_time:
      total_cpu_time = cpu_time[-1] - first_sample
    #Else, we add the last sample to the total CPU consumption time
    else:
134 135
      total_cpu_time = total_cpu_time + cpu_time[-1]

136
    return [total_cpu_time, sum(memory_space) / float(len(memory_space)), start_time, end_time, start_date, end_date]
Łukasz Nowak's avatar
Łukasz Nowak committed
137 138 139 140 141 142
    #return [scipy.mean(cpu_percentage), cpu_time, scipy.mean(memory_percentage),
    #      scipy.mean(memory_space), start_time, end_time, start_date, end_date]



  def _write_xml_output(self, res_list, storage_path):
143
    """This function provide to dump on xml the consumption infos,
Łukasz Nowak's avatar
Łukasz Nowak committed
144 145
    the res_list contains the following informations:

146
    [CPU mean %, CPU whole usage (seconds), Memory mean %, Memory mean space usage (byte),
Łukasz Nowak's avatar
Łukasz Nowak committed
147 148 149 150
    start_time, end_time, start_date, end_date]"""

    #XXX- NOTE

151
    """The res_list has been recently changed, now it contains
Łukasz Nowak's avatar
Łukasz Nowak committed
152
    [CPU whole usage (seconds), Memory mean space usage (byte)]"""
153

Łukasz Nowak's avatar
Łukasz Nowak committed
154 155 156

    res_list = map(str, res_list)

157 158
    cpu_list = ['CPU Consumption',
                'CPU consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
159
                res_list[0],
Łukasz Nowak's avatar
Łukasz Nowak committed
160 161
                ]

162
    memory_list = ['Memory consumption',
163
                  'Memory consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
Łukasz Nowak's avatar
Łukasz Nowak committed
164 165 166
                  res_list[1],
                  ]

167 168 169 170 171 172 173 174 175 176
    root = ElementTree.Element("consumption")
    #Then, we add two movement elements, one for cpu
    tree = self._add_movement(root, cpu_list )
    #And one for memory
    tree = self._add_movement(root, memory_list)

    #We add the XML header to the file to be valid
    report = ElementTree.tostring(tree)
    fd = open(storage_path, 'w')
    fd.write("<?xml version='1.0' encoding='utf-8'?>%s" % report)
177 178
    fd.close()

179
  def _add_movement(self, consumption, single_resource_list):
180

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
    child_consumption = ElementTree.SubElement(consumption, "movement")

    child_movement = ElementTree.SubElement(child_consumption, "resource")
    child_movement.text = single_resource_list[0]
    child_movement = ElementTree.SubElement(child_consumption, "title")
    child_movement.text = single_resource_list[1]
    child_movement = ElementTree.SubElement(child_consumption, "reference")
    child_movement.text = ''
    child_movement = ElementTree.SubElement(child_consumption, "quantity")
    child_movement.text = single_resource_list[2]
    child_movement = ElementTree.SubElement(child_consumption, "price")
    child_movement.text = '0.00'
    child_movement = ElementTree.SubElement(child_consumption, "VAT")
    child_movement.text = ''
    child_movement = ElementTree.SubElement(child_consumption, "category")
    child_movement.text = ''

    tree = self.element_tree.ElementTree(consumption)
Łukasz Nowak's avatar
Łukasz Nowak committed
199 200 201 202 203 204
    return tree


def parse_opt():

  usage="""usage: slapmonitor [options] PID_FILE_PATH DATABASE_PATH
205
Usage: slapreport [options] LOG_PATH DATABASE_PATH LOGBOX_IP LOGBOX_PORT LOGBOX_LOGIN LOGBOX_PASSWORD"""
Łukasz Nowak's avatar
Łukasz Nowak committed
206

207
  parser = optparse.OptionParser(usage=usage)
Łukasz Nowak's avatar
Łukasz Nowak committed
208 209 210 211 212 213 214
  parser.add_option('-t', '--update_time',type=int, dest='update_time', help='Specify the interval'\
                    '(in seconds) to check resources consumption [default 30 seconds]', default=3)
  parser.add_option('-l', '--log_file', dest='path_log_file',help='Specify the logfile destination path',
                    metavar='FILE')
  return parser

class SlapMonitor(object):
215

Łukasz Nowak's avatar
Łukasz Nowak committed
216 217 218 219
  def __init__(self, proc, update_time, path_database):
    self.proc = proc
    self.update_time = update_time
    self.path_database = path_database
220 221
    self.start_monitor()

Łukasz Nowak's avatar
Łukasz Nowak committed
222 223

  def start_monitor(self):
224

Łukasz Nowak's avatar
Łukasz Nowak committed
225 226 227 228 229 230
    temporary_monitoring = MonitoringTool()
    #check if the process is alive == None, instead zero value == proc is over
    while self.proc.pid in psutil.get_pid_list():
      conn = sqlite3.connect(self.path_database)
      cursor = conn.cursor()
      cursor.execute("create table if not exists data (cpu real, cpu_time real, memory real, rss real," \
Mohamadou Mbengue's avatar
Mohamadou Mbengue committed
231
                      "date text, time text, reported integer NULL DEFAULT 0)")
Łukasz Nowak's avatar
Łukasz Nowak committed
232 233 234 235 236 237
      try:
        res_list = temporary_monitoring.get_cpu_and_memory_usage(self.proc)
        date_catched = "-".join([str(res_list[4].year), str(res_list[4].month), str(res_list[4].day)])
        time_catched = ":".join([str(res_list[4].hour), str(res_list[4].minute), str(res_list[4].second)])
        res_list[4] = date_catched
        res_list.append(time_catched)
238
        cursor.execute("insert into data(cpu, cpu_time, memory, rss, date, time) values (?,?,?,?,?,?)" , res_list)
Łukasz Nowak's avatar
Łukasz Nowak committed
239 240 241 242 243
        conn.commit()
        cursor.close()
        conn.close()
        time.sleep(self.update_time)
      except IOError:
244 245
        if log_file:
          logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
246
                % (self.proc.pid, time.strftime("%Y-%m-%d at %H:%m")))
247
        sys.exit(1)
Łukasz Nowak's avatar
Łukasz Nowak committed
248 249 250 251
    if log_file:
      logging.info("EXIT 0: Process terminated normally!")
    sys.exit(0)

252 253 254

class SlapReport(object):
  """
255
  reports usage of apache and mariadb logs to an existing log server
256 257 258 259 260 261 262
  """
  def __init__(self, proc, update_time, consumption_log_path, database_path, ssh_parameters):
    self.proc = proc
    self.update_time = update_time
    self.path_database = database_path
    self.consumption_log_path = consumption_log_path
    self.ssh_parameters = ssh_parameters
263
    self.start_report()
264 265 266 267 268 269 270 271 272

  def write_log(self):
    """This func read data from database and through
       write_log_file write result on a log file"""

    #write none reported log in the consumption log log file
    fd = open(self.consumption_log_path, 'a')
    conn = sqlite3.connect(self.path_database)
    cursor = conn.cursor()
Mohamadou Mbengue's avatar
Mohamadou Mbengue committed
273
    log_list = cursor.execute("SELECT * FROM data WHERE reported=0 ORDER BY rowid ASC")
274 275 276
    last_report_time = ""
    for row in log_list:
      cpu_consumption_info = "%s %s Instance %s CPU Consumption: CPU:%s CPU_TIME:%s\n" \
277
          % (row[4],row[5], self.proc.name, row[0],row[1])
278
      memory_consumption_info = "%s %s Instance %s Memory Consumption: %s\n" \
279
          % (row[4], row[5], self.proc.name, row[2])
280
      try:
281 282 283 284
        cmd = "tail -n 2 %s | ssh %s:%s@%s -p %s " \
            % (self.consumption_log_path, self.ssh_parameters['user'], \
            self.ssh_parameters['passwd'], self.ssh_parameters['ip'], \
            self.ssh_parameters['port'])
285
        res = os.system(cmd)
286
        if not res:
287 288 289
          if last_report_time != row[5]:
            fd.write("%s%s" % (cpu_consumption_info,memory_consumption_info))
            last_report_time = "%s" % row[5]
Mohamadou Mbengue's avatar
Mohamadou Mbengue committed
290
          cursor.execute("UPDATE data set reported='1' WHERE time=?", (row[5],))
291 292 293
        conn.commit()

      except Exception:
294 295
        if log_file:
          logging.info("ERROR : Unable to connect to % at %s"
296
                       % (self.ssh_parameters['ip'], time.strftime("%Y-%m-%d at %H:%m")))
297 298 299 300 301 302 303
          #sys.exit(1)

    cursor.close()
    conn.close()

  def start_report(self):
    """
304
    while the process is running, connect to the database
305 306 307 308 309
    and report the non-reported line,
    when line is reported put 1 to column report
    """
    temporary_monitoring = MonitoringTool()
    #check if the process is alive == None, instead zero value == proc is over
310
    while self.proc.pid in psutil.get_pid_list():
311
      try:
312
        # send log to logbox
313
        self.write_log()
Mohamadou Mbengue's avatar
Mohamadou Mbengue committed
314
        time.sleep(self.update_time)
315
      except IOError:
316 317
        if log_file:
          logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
318
                       % (self.proc.pid, time.strftime("%Y-%m-%d at %H:%m")))
319
          sys.exit(1)
320 321 322 323
    if log_file:
      logging.info("EXIT 0: Process terminated normally!")
    sys.exit(0)

Łukasz Nowak's avatar
Łukasz Nowak committed
324 325 326 327 328 329 330 331 332
def run_slapmonitor():
  #This function require the pid file and the database path
  parser = parse_opt()
  opts, args = parser.parse_args()
  if len(args) != 2:
    parser.error("Incorrect number of arguments, 2 required but "+str(len(args))+" detected" )

  if opts.path_log_file:
    logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
333
    global log_file
Łukasz Nowak's avatar
Łukasz Nowak committed
334
    log_file = True
335

336
  proc = psutil.Process(read_pid(args[0]))
337 338
  # XXX FIXME: THE PID IS ONLY READ ONCE.
  # process death and pid reuse are not detected.
Łukasz Nowak's avatar
Łukasz Nowak committed
339
  SlapMonitor(proc, opts.update_time, args[1])
340

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358

def run_slapmonitor_xml():
  #This function require the database path and XML path
  parser = parse_opt()
  opts, args = parser.parse_args()
  if len(args) != 2:
    parser.error("Incorrect number of arguments, 2 required but "+str(len(args))+" detected" )

  if opts.path_log_file:
    logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
    global log_file
    log_file = True

  get_xml_hand = GenerateXML(ElementTree, args[0], args[1])
  get_xml_hand.dump_xml()



359
def run_slapreport_():
Łukasz Nowak's avatar
Łukasz Nowak committed
360 361 362 363 364 365 366 367 368 369 370 371 372 373
  #This function require the xml_path and database_path
  parser = parse_opt()
  opts, args = parser.parse_args()
  if len(args) != 2:
    parser.error("Incorrect number of arguments, 2 required but "+str(len(args))+" detected" )

  if opts.path_log_file:
    logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
    global log_file
    log_file = True

  get_xml_hand = GenerateXML(ElementTree, args[1], args[0])
  get_xml_hand.dump_xml()

374 375 376 377
def run_slapreport():
  #This function require the consumption_log_path and database_path ssh_parameters
  parser = parse_opt()
  ssh_parameters ={}
378
  opts, args = parser.parse_args()
379 380 381 382 383 384 385 386

  if len(args) != 7:
    parser.error("Incorrect number of arguments, 7 required but "+str(len(args))+" detected" )

  if opts.path_log_file:
    logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
    global log_file
    log_file = True
387

388
  if args[3] == "":
389
    sys.exit(0)
390

391 392 393 394 395 396 397 398
  pid_file_path = "%s" % args[0]
  #set ssh parameters
  ssh_parameters['ip']=args[3]
  ssh_parameters['port']=args[4]
  ssh_parameters['user']=args[5]
  ssh_parameters['passwd']=args[6]

  try:
399
    proc = psutil.Process(read_pid(pid_file_path))
400 401 402 403
    SlapReport(proc, opts.update_time, args[1], args[2], ssh_parameters)
  except IOError:
    if log_file:
      logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
404
                   % (proc.pid, time.strftime("%Y-%m-%d at %H:%m")))
405
      sys.exit(1)
406

407 408 409
  if log_file:
    logging.info("EXIT 0: Process terminated normally!")
  #sys.exit(0)