Commit ed17ad18 authored by Hardik Juneja's avatar Hardik Juneja Committed by Rafael Monnerat

make monitor.collect use slapos.collect.db API's

parent 9ce3279a
...@@ -37,6 +37,8 @@ import psutil ...@@ -37,6 +37,8 @@ import psutil
from time import strftime from time import strftime
from datetime import datetime, timedelta from datetime import datetime, timedelta
from slapos.collect.db import Database
from slapos.collect.reporter import ConsumptionReportBase
def parseArguments(): def parseArguments():
""" """
...@@ -64,45 +66,15 @@ class ResourceCollect: ...@@ -64,45 +66,15 @@ class ResourceCollect:
def __init__(self, db_path = None): def __init__(self, db_path = None):
# XXX this code is duplicated with slapos.collect.db.Database.__init__ # XXX this code is duplicated with slapos.collect.db.Database.__init__
assert os.path.exists(db_path) and os.path.isfile(db_path) assert os.path.exists(db_path)
self.uri = db_path if db_path.endswith("collector.db"):
self.connection = None db_path = db_path[:-len("collector.db")]
self.cursor = None self.db = Database(db_path)
self.consumption_utils = ConsumptionReportBase(self.db)
def connect(self):
# XXX this code is duplicated with slapos.collect.db.Database.connect
self.connection = sqlite3.connect(self.uri)
self.cursor = self.connection.cursor()
def close(self):
# XXX this code is duplicated with slapos.collect.db.Database.connect
assert self.connection is not None
self.cursor.close()
self.connection.close()
def _execute(self, sql):
# XXX this code is duplicated with slapos.collect.db.Database.connect
assert self.connection is not None
return self.cursor.execute(sql)
def select(self, table, date=None, columns="*", where=None):
""" Query database for a full table information """
# XXX this code is duplicated with slapos.collect.db.Database.select
if date is not None:
where_clause = " WHERE date = '%s' " % date
else:
where_clause = ""
if where is not None:
if where_clause == "":
where_clause += " WHERE 1 = 1 "
where_clause += " AND %s " % where
select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause)
return self._execute(select_sql)
def has_table(self, name): def has_table(self, name):
self.connect() self.db.connect()
check_result_cursor = self.select( check_result_cursor = self.db.select(
table="sqlite_master", table="sqlite_master",
columns='name', columns='name',
where="type='table' AND name='%s'" % name) where="type='table' AND name='%s'" % name)
...@@ -112,89 +84,36 @@ class ResourceCollect: ...@@ -112,89 +84,36 @@ class ResourceCollect:
return True return True
def getPartitionCPULoadAverage(self, partition_id, date_scope): def getPartitionCPULoadAverage(self, partition_id, date_scope):
# XXX Code seems copied from slapos.collect.report.ConsumptionReport._getPartitionCPULoadAverage return self.consumption_utils.getPartitionCPULoadAverage(partition_id, date_scope)
self.connect()
query_result_cursor = self.select("user", date_scope,
columns="SUM(cpu_percent)",
where="partition = '%s'" % partition_id)
cpu_percent_sum = zip(*query_result_cursor)
if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
return
query_result_cursor = self.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.close()
if len(sample_amount) and len(cpu_percent_sum):
return round(cpu_percent_sum[0][0]/sample_amount[0][0], 2)
def getPartitionUsedMemoryAverage(self, partition_id, date_scope): def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
# XXX Code seems copied from slapos.collect.report.ConsumptionReport._getPartitionUsedMemoryAverage return self.consumption_utils.getPartitionUsedMemoryAverage(partition_id, date_scope)/(1024*1024.0)
self.connect()
query_result_cursor = self.select("user", date_scope,
columns="SUM(memory_rss)",
where="partition = '%s'" % partition_id)
memory_sum = zip(*query_result_cursor)
if len(memory_sum) and memory_sum[0][0] is None:
return
query_result_cursor = self.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.close()
if len(sample_amount) and len(memory_sum):
return round(memory_sum[0][0]/(sample_amount[0][0]*1024*1024.0), 2)
def getPartitionDiskUsedAverage(self, partition_id, date_scope): def getPartitionDiskUsedAverage(self, partition_id, date_scope):
# XXX Code seems copied from slapos.collect.report.ConsumptionReport._getPartitionDiskUsedAverage return self.consumption_utils.getPartitionDiskUsedAverage(partition_id, date_scope)/1024.0
if not self.has_table('folder'):
return
self.db.connect()
query_result_cursor = self.select("folder", date_scope,
columns="SUM(disk_used)",
where="partition = '%s'" % partition_id)
disk_used_sum = zip(*query_result_cursor)
if len(disk_used_sum) and disk_used_sum[0][0] is None:
return
query_result_cursor = self.select("folder", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
collect_amount = zip(*query_result_cursor)
self.db.close()
if len(collect_amount) and len(disk_used_sum):
return round(disk_used_sum[0][0]/(collect_amount[0][0]*1024.0), 2)
def getPartitionConsumption(self, partition_id, where=""): def getPartitionConsumption(self, partition_id, where="", date_scope=None, min_time=None, max_time=None):
""" """
Query collector db to get consumed resource for last minute Query collector db to get consumed resource for last minute
""" """
self.connect() self.db.connect()
comsumption_list = [] comsumption_list = []
if where != "": if where != "":
where = "and %s" % where where = "and %s" % where
if not date_scope:
date_scope = datetime.now().strftime('%Y-%m-%d') date_scope = datetime.now().strftime('%Y-%m-%d')
if not min_time:
min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00') min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
if not max_time:
max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59') max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
sql_query = """select count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time), columns = """count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time),
MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid, SUM(io_rw_counter), MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid, SUM(io_rw_counter),
SUM(io_cycles_counter) from user SUM(io_cycles_counter)"""
where date='%s' and partition='%s' and (time between '%s' and '%s') %s query_result = self.db.select("user", date_scope, columns,
group by pid order by cpu_result desc""" % ( where="partition = '%s' and (time between '%s' and '%s') %s" %
date_scope, partition_id, min_time, max_time, where) (partition_id, min_time, max_time, where),
query_result = self._execute(sql_query) group="pid", order="cpu_result desc")
for result in query_result: for result in query_result:
count = int(result[0]) count = int(result[0])
if not count > 0: if not count > 0:
...@@ -219,22 +138,25 @@ group by pid order by cpu_result desc""" % ( ...@@ -219,22 +138,25 @@ group by pid order by cpu_result desc""" % (
resource_dict['user'] = pprocess.username() resource_dict['user'] = pprocess.username()
resource_dict['date'] = datetime.fromtimestamp(pprocess.create_time()).strftime("%Y-%m-%d %H:%M:%S") resource_dict['date'] = datetime.fromtimestamp(pprocess.create_time()).strftime("%Y-%m-%d %H:%M:%S")
comsumption_list.append(resource_dict) comsumption_list.append(resource_dict)
self.close() self.db.close()
return comsumption_list return comsumption_list
def getPartitionComsumptionStatus(self, partition_id, where=""): def getPartitionComsumptionStatus(self, partition_id, where="", date_scope=None, min_time=None, max_time=None):
self.connect() self.db.connect()
if where != "": if where != "":
where = " and %s" % where where = " and %s" % where
if not date_scope:
date_scope = datetime.now().strftime('%Y-%m-%d') date_scope = datetime.now().strftime('%Y-%m-%d')
if not min_time:
min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00') min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
if not max_time:
max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59') max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
sql_query = """select count(pid), SUM(cpu_percent), SUM(cpu_time),
SUM(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), SUM(io_rw_counter), colums = """count(pid), SUM(cpu_percent), SUM(cpu_time), SUM(cpu_num_threads), SUM(memory_percent),
SUM(io_cycles_counter) from user where SUM(memory_rss), SUM(io_rw_counter), SUM(io_cycles_counter)"""
date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % ( query_result = self.db.select('user', date_scope, colums,
date_scope, partition_id, min_time, max_time, where) where="partition='%s' and (time between '%s' and '%s') %s" %
query_result = self._execute(sql_query) (partition_id, min_time, max_time, where))
result_list = zip(*query_result) result_list = zip(*query_result)
process_dict = memory_dict = io_dict = {} process_dict = memory_dict = io_dict = {}
...@@ -256,7 +178,7 @@ date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % ( ...@@ -256,7 +178,7 @@ date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % (
'date': '%s %s' % (date_scope, min_time) 'date': '%s %s' % (date_scope, min_time)
} }
if self.has_table('folder'): if self.has_table('folder'):
disk_result_cursor = self.select( disk_result_cursor = self.db.select(
"folder", date_scope, "folder", date_scope,
columns="SUM(disk_used)", columns="SUM(disk_used)",
where="partition='%s' and (time between '%s' and '%s') %s" % ( where="partition='%s' and (time between '%s' and '%s') %s" % (
...@@ -267,7 +189,7 @@ date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % ( ...@@ -267,7 +189,7 @@ date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % (
disk_used_sum = zip(*disk_result_cursor) disk_used_sum = zip(*disk_result_cursor)
if len(disk_used_sum) and disk_used_sum[0][0] is not None: if len(disk_used_sum) and disk_used_sum[0][0] is not None:
io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2) io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2)
self.close() self.db.close()
return (process_dict, memory_dict, io_dict) return (process_dict, memory_dict, io_dict)
def appendToJsonFile(file_path, content, stepback=2): def appendToJsonFile(file_path, content, stepback=2):
......
...@@ -50,7 +50,6 @@ def main(): ...@@ -50,7 +50,6 @@ def main():
args = parser.parse_args() args = parser.parse_args()
# get last minute # get last minute
partition = args.partition.replace('part', 'user')
now = datetime.datetime.now() now = datetime.datetime.now()
currentdate = now.strftime('%Y-%m-%d') currentdate = now.strftime('%Y-%m-%d')
delta = datetime.timedelta(minutes=1) delta = datetime.timedelta(minutes=1)
......
##############################################################################
#
# Copyright (c) 2017 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import os
import sqlite3
import time
from slapos.test.promise import data
from slapos.monitor.collect import ResourceCollect
class TestMonitorCollect(unittest.TestCase):
def setUp(self):
self.base_path = "/".join(data.__file__.split("/")[:-1])
self.status = "ok"
# populate db
self.conn = sqlite3.connect('/tmp/collector.db')
f = open(self.base_path+"/monitor_collect.sql")
sql = f.read()
self.conn.executescript(sql)
self.conn.close()
# inititalise
self.collector = ResourceCollect('/tmp/')
def test_getPartitionUsedMemoryAverage(self):
self.assertEquals(1195.492578125,
self.collector.getPartitionUsedMemoryAverage('slapuser15', '2017-09-16'))
def test_getPartitionCPULoadAverage(self):
self.assertEquals(2.1599999999999993,
self.collector.getPartitionCPULoadAverage('slapuser15', '2017-09-16'))
def test_getPartitionDiskUsedAverage(self):
self.assertEquals(35.5234375,
self.collector.getPartitionDiskUsedAverage('slapuser15', '2017-04-18'))
def test_getPartitionConsumption(self):
data = self.collector.getPartitionConsumption('slapuser15', date_scope='2017-09-16',
min_time='00:01:00', max_time='00:13:00')
self.assertEquals(1302.66, data[0]['cpu_time'])
self.assertEquals(26825304064.0, data[0]['io_rw_counter'])
def test_getPartitionComsumptionStatus(self):
data = self.collector.getPartitionComsumptionStatus('slapuser15', date_scope='2017-09-16',
min_time='00:01:00', max_time='00:13:00')
self.assertEquals(7.3, data[0]['cpu_percent'])
self.assertEquals(2822535483392.0, data[2]['io_rw_counter'])
def tearDown(self):
os.remove("/tmp/collector.db")
if __name__ == '__main__':
unittest.main()
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment