implemented nexedi's slapos python monitor for ram

Milestone1/results/test.csv

-items,ramUsed,memUsed
-1000,0.1296115260144386,5.12109375
-2000,0.1690131572551702,7.19921875
-3000,0.1530372945722992,7.69921875
-4000,0.1568615260144386,15.44921875
-5000,0.13156515504669675,9.7265625
-6000,-0.1358904734856864,12.16015625
-7000,-0.10511908929321523,12.6328125
-8000,0.13174770641252376,1.88671875
-9000,0.024319474027435395,2.359375
-10000,-0.14388847398556137,4.86328125
-11000,-0.14388847398556137,5.19140625
-12000,-0.11240421611449691,25.125
-13000,-0.14388847398556137,8.59375
-14000,-0.13589247198656085,30.96875
-15000,-0.14012435855268812,32.8828125
-16000,-0.10031164778908785,14.68359375
-17000,-0.1358904734856864,14.45703125
-18000,-0.14388847398556137,17.10546875
-19000,-0.13987341752382987,23.25
-20000,-0.13992022001730742,19.984375
-21000,-0.1283845980165691,20.68359375
-22000,-0.1360740046132397,23.18359375
-23000,-0.14388847398556137,35.23828125
-24000,0.4067745232623746,38.765625
-25000,-0.13589247198656085,41.1875
-26000,-0.11687496723218471,29.30078125
-27000,-0.14388847398556137,47.12890625
-28000,-0.1083973467673659,50.73828125
-29000,-0.13988847398556137,32.81640625
-30000,-0.13188847398556136,35.2421875
-31000,-0.14388847398556137,35.77734375
-32000,-0.12072533575392996,40.8125
-33000,-0.14388847398556137,38.6015625
-34000,-0.13641463243100238,41.10546875
-35000,-0.12789646998756032,41.9921875
-36000,-0.13993590481560092,44.48046875
-37000,-0.13985621592104525,44.88671875
-38000,-0.13988847398556137,47.390625
-39000,-0.13610734948756942,50.19140625
-40000,-0.14388847398556137,52.73828125
-41000,-0.14388847398556137,53.13671875
-42000,-0.14388847398556137,76.7109375
-43000,-0.12398421598606524,56.046875
-44000,-0.13988847398556137,58.65234375
-45000,-0.14388847398556137,59.11328125
-46000,-0.12874824067704804,61.578125
-47000,-0.14388847398556137,61.9453125
-48000,-0.14388847398556137,64.4296875
-49000,-0.12398421598606524,64.80078125
-50000,-0.14388847398556137,65.515625
-51000,-0.13988847398556137,67.73828125
-52000,-0.13582598973260096,68.55078125
-53000,-0.13189447098706064,71.22265625
-54000,-0.14013847398556134,71.97265625
-55000,-0.1401375362511278,74.37109375
-56000,-0.13988947373562388,75.12109375
-57000,-0.12389347273587381,77.25
-58000,-0.14388847398556137,78.203125
-59000,-0.14388847398556137,80.20703125
-60000,-0.13586037564135667,81.671875
-61000,-0.13988947373562388,83.16015625
-62000,-0.11276297597759322,85.16015625
-63000,-0.13613653600106523,86.25390625
-64000,-0.13988947373562388,86.79296875
-65000,-0.1358904734856864,89.23046875
-66000,-0.07037738502468427,89.8046875
-67000,-0.09356460751819717,92.29296875
-68000,-0.13189447098706064,93.34375
-69000,-0.09739400904091189,95.765625
-70000,-0.14388847398556137,98.18359375
-71000,-0.14388847398556137,99.09765625
-72000,-0.13213847398556133,99.515625
-73000,-0.1323827211091232,102.07421875
-74000,-0.13988847398556137,102.51953125
-75000,-0.13209424562922512,104.9453125
-76000,-0.13607794236550597,105.47265625
-77000,-0.13988847398556137,107.9921875
-78000,-0.13592034649552154,108.4453125
-79000,-0.14388847398556137,110.9453125
-80000,-0.14388847398556137,111.51953125
-81000,-0.1213173258010668,114.53125
-82000,-0.08535920935324526,119.125
-83000,-0.12454323589032323,117.4296875
-84000,-0.04616290537771328,126.203125
-85000,-0.14388847398556137,120.46484375
-86000,-0.06986996935940482,127.68359375
-87000,-0.14388847398556137,123.48828125
-88000,-0.11618066542132965,133.29296875
-89000,-0.13214141075136987,126.46484375
-90000,-0.09275497272611555,140.37109375
-91000,-0.13988847398556137,129.609375
-92000,-0.09695072298154528,146.25
-93000,-0.1401375362511278,133.18359375
-94000,-0.06502729988629208,143.36328125
-95000,-0.13238559826663165,128.60546875
-96000,-0.0538659683591548,138.01953125
-97000,-0.13214141075136987,131.44921875
-98000,-0.1401375362511278,154.03125
-99000,-0.1361053959037135,134.48828125
-100000,-0.1282012229895454,134.95703125
-101000,-0.12446132579128988,137.3828125
-102000,-0.13641463243100238,137.9140625
-103000,-0.13613847398556134,161.6875
-104000,-0.1321355357510028,160.46875
-105000,-0.13613847398556134,143.98046875
-106000,-0.13613847398556134,144.52734375
-107000,-0.14388847398556137,146.88671875
-108000,-0.12421715924452548,150.640625
-109000,-0.14388847398556137,155.12109375
-110000,-0.09293465872451723,165.87109375
-111000,-0.14388847398556137,158.32421875
-112000,-0.08138847398556137,163.21875
-113000,-0.12414834405052888,162.3671875
-114000,-0.06959128523054126,167.07421875
-115000,-0.14388847398556137,165.30078125
-116000,-0.13614041100130747,182.70703125
-117000,-0.14388847398556137,168.34375
-118000,-0.13218237186850035,168.77734375
-119000,-0.14388847398556137,171.35546875
-120000,-0.06986996935940482,178.375
-121000,-0.12439334776711597,192.6484375
-122000,-0.12813847398556133,195.9921875
-123000,-0.14388847398556137,177.83984375
-124000,-0.13988947373562388,178.23828125
-125000,-0.0897941811815911,201.44921875
-126000,-0.14388847398556137,181.03515625
-127000,-0.1358904734856864,183.5390625
-128000,-0.1130090273452452,197.55859375
-129000,-0.13988947373562388,186.63671875
-130000,-0.13613653600106523,187.33984375
-131000,-0.11663847398556138,202.4375
-132000,-0.1363865985166941,190.546875
-133000,-0.1363865985166941,193.1875
-134000,-0.10479484848755338,193.6875
-135000,-0.13988847398556137,196.19921875
-136000,-0.12844638183114043,196.625
-137000,-0.13607597398556137,199.203125
-138000,-0.13588847398556136,199.64453125
-139000,-0.13988947373562388,220.984375
-140000,-0.11670784636015985,203.09765625
-141000,-0.1401375362511278,205.71484375
-142000,-0.11239439487243463,205.97265625
-143000,-0.1005833470567411,214.73828125
-144000,-0.13218237186850035,209.21875
-145000,-0.14388847398556137,211.703125
-146000,-0.13613847398556134,212.21484375
-147000,-0.13613847398556134,215.234375
-148000,-0.13586037564135667,215.63671875
-149000,-0.14388847398556137,218.3203125
-150000,-0.13582598973260096,218.82421875
-151000,-0.14388847398556137,221.07421875
-152000,-0.13592233009708743,217.7265625
-153000,-0.14388847398556137,241.40234375
-154000,-0.1398724097285332,220.7890625
-155000,-0.14388847398556137,223.71484375
-156000,-0.08515315781460409,232.1796875
-157000,-0.1401082320500775,226.8046875
-158000,-0.05036509314035009,230.4375
-159000,-0.1358904734856864,229.8125
-160000,-0.10072661702194785,240.390625
-161000,-0.13184632646925432,254.1796875
-162000,-0.13613653600106523,233.92578125
-163000,-0.1401375362511278,236.30078125
-164000,-0.14388847398556137,236.89453125
-165000,-0.13218820014936516,239.2578125
-166000,-0.13616935047958523,239.86328125
-167000,-0.14388847398556137,257.89453125
-168000,-0.14388847398556137,242.93359375
-169000,-0.12863466053219808,245.3515625
-170000,-0.1321355357510028,265.66796875
-171000,-0.13610734948756942,249.0234375
-172000,-0.13613653600106523,249.43359375
-173000,-0.14388847398556137,251.83203125
-174000,-0.07763847398556134,260.19140625
-175000,-0.12438847398556141,254.87890625
-176000,-0.13238559826663165,276.62109375
-177000,-0.13614041100130747,258.58984375
-178000,-0.1361961662932537,258.96484375
-179000,-0.12438359776650665,261.34375
-180000,-0.11664528478286207,282.90625
-181000,-0.12813847398556133,264.48828125
-182000,-0.14388847398556137,265.14453125
-183000,-0.1401082320500775,267.5234375
-184000,-0.13238559826663165,289.63671875
-185000,-0.14388847398556137,270.98828125
-186000,-0.09336321135424575,276.125
-187000,-0.12789646998756032,274.109375
-188000,-0.09713847398556141,282.890625
-189000,-0.12813453550094023,299.31640625
-190000,-0.09688847398556133,294.0078125
-191000,-0.14388847398556137,281.4296875
-192000,-0.11639534726724099,298.98046875
-193000,-0.1321852867345653,284.96875
-194000,-0.07686907363282769,294.05859375
-195000,-0.14388847398556137,288.05859375
-196000,0.036332409548575195,306.92578125
-197000,-0.13242858360190413,291.3359375
-198000,-0.08516783430540142,319.05078125
-199000,-0.14388847398556137,295.60546875
-200000,0.020570411293118984,317.1484375
-201000,-0.14388847398556137,299.01953125
-202000,-0.12423121592104525,317.1875
-203000,-0.13613847398556134,323.421875
-204000,-0.0891384739855614,305.05078125
-205000,-0.13613847398556134,327.22265625
-206000,-0.11190660543859587,306.375
-207000,-0.12825571964312965,308.7578125
-208000,-0.10887972179751437,309.94140625
-209000,-0.13988947373562388,328.1171875
-210000,-0.1163884739855614,325.6796875
-211000,-0.13214141075136987,329.4921875
-212000,-0.10873026203176928,336.23828125
-213000,-0.14388847398556137,318.30859375
-214000,-0.1280791891674936,334.7890625
-215000,-0.13607597398556137,321.21484375
-216000,-0.10887972179751437,356.40625

Milestone1/run.js

@@ -19,6 +19,7 @@ const args = require("yargs")
     .nargs("verbose", 0)
     .argv;
 
+
 var run;
 var to_run = [];
 var options = [];
@@ -82,6 +83,7 @@ run = function (callback) {
 };
 
 fs.readFile("./config.json", "utf8", function (err, data) {
+
     if (err) {
         return console.log("Error while reading config file : " + err);
     }

Milestone1/unit_tests/add_csv.js

@@ -2,9 +2,9 @@
 
 var childProcess = require("child_process");
 var cut_file = require("./random_gen").cut_file;
-var diskspace = require("diskspace");
+// var diskspace = require("diskspace");
 // var os_utils = require("os-utils");
-var os = require('os');
+// var os = require('os');
 const puppeteer = require("puppeteer");
 const Server = require("ws").Server;
 const args = require("yargs")
@@ -28,8 +28,9 @@ var page;
 var n = 0;
 var time;
 var monitor;
-var ramBefore;
-var memBefore;
+// var ramBefore;
+// var memBefore;
+var browser_pid;
 
 function init_server (){
     var port = 9030;
@@ -92,10 +93,11 @@ async function query (){
     files = cut_file(args.file, 1000);
     init_server();
     browser = await puppeteer.launch();
+    browser_pid = browser.process().pid;
     page = await browser.newPage();
     await page.goto(link);
     monitor = childProcess.fork("./unit_tests/monitor.js",
-        args.r ? ["-r"] : []);
+        args.r ? ["-r", "-p", browser_pid] : ["-p", browser_pid]);
     timeout();
     query();
     // diskspace.check("C", function (err, result){

Milestone1/unit_tests/monitor.js

-var os = require('os');
 const fs = require("fs");
 var diskspace = require("diskspace");
 const args = require("yargs").argv;
+var childProcess = require("child_process");
 
+var pid = args.p;
 var path = args.r
     ? "./results/add_random_csv_monitor.csv"
     : "./results/add_csv_monitor.csv";
@@ -10,35 +11,36 @@ var stream = fs.createWriteStream(path, {flags: "w+"});
 stream.write("time,ram,disk\n");
 
 var start_disk;
-var start_ram = os.totalmem() - os.freemem();
-
-// function init_server (){
-//     var port = 9030;
-//     var ws = new Server({port: port});
-//     ws.on("connection", function(w){
-//         w.on("message", function(){
-//             process.send(max_ram);
-//         });
-//         w.on("close", function() {
-//             ws.close();
-//         });
-//     });
-// }
-
-var check = function(){
-    setTimeout(function () {
-        diskspace.check("/", function(err, result){
-            //var disk_used = parseInt(result.used) - parseInt(start_disk);
-            var disk_used = result.used - start_disk;
-            var ram_used = os.totalmem() - os.freemem();
-            stream.write(Date.now() + "," + ram_used + "," + disk_used + "\n");
-            check();
-        });
-    }, 1);
+
+function init_monitor(){
+  var net = require('net');
+  const port = 9040;
+
+  var server = net.createServer(function(socket) {
+    socket.on("data", function(data){
+      diskspace.check("/", function(err, result){
+          var disk_used = result.used - start_disk;
+          var ram_used = data;
+          stream.write(Date.now() + "," + ram_used + "," + disk_used + "\n");
+      });
+    })
+    socket.on("error", function(err){
+      console.log("Error : " + err.message);
+    })
+  });
+
+  server.listen(port, function() {});
+
+  var monitor = childProcess.spawn('python',
+    ["./unit_tests/monitor.py", "--p", pid],
+    { stdio: 'inherit' });
 }
 
-//init_server();
+process.on("exit", function(){
+  monitor.kill();
+})
+
 diskspace.check("C", function (err, result){
     start_disk = result.used;
-    check();
+    init_monitor();
 });

Milestone1/unit_tests/monitor.py

+import psutil
+import os
+import subprocess
+import time
+
+import argparse
+parser = argparse.ArgumentParser(description='')
+parser.add_argument("--p", type=int, help="pid")
+args = parser.parse_args()
+
+import socket
+client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+client.connect(('localhost', 9040))
+
+class _Snapshot(object):
+  def get(self, property, default=None):
+    return getattr(self, property, default)
+
+class ProcessSnapshot(_Snapshot):
+  def __init__(self, process=None):
+    assert type(process) is psutil.Process
+    self.username = process.username()
+    self.process_object = process
+    self.pid = process.pid
+    # Memory percentage
+    self.memory_percent = process.memory_percent()
+
+  def update_memory_percent(self):
+    if self.process_object.is_running():
+      self.memory_percent = self.process_object.memory_percent()
+
+process = psutil.Process(args.p)
+snapshot = ProcessSnapshot(process)
+
+while (1) :
+    snapshot.update_memory_percent()
+    client.send(str(snapshot.memory_percent))
+    time.sleep(0.1)