final rerun of all tests

Milestone1/bench_files/add_csv_attachments_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
-fs.truncateSync("./results/add_csv_attachments.csv");
+
+var stream = fs.createWriteStream("../results/add_csv_attachments.csv", {flags:'a'});
+fs.truncateSync("../results/add_csv_attachments.csv");
 stream.write("items,ramUsed,memUsed\n");
 
 var to_run = [];
-fs.readdir("./files/CSV/", function(err, items){
+fs.readdir("../files/CSV/", function(err, items){
   if (err) {
     return console.log('Unable to scan directory: ' + err);
   }
   items.forEach(function (file) {
-    to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "attachment"], amount : parseInt(file.substring(17, file.length-4))});
+    to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "attachment"]});
+  });
+  to_run.sort(function(file1, file2){
+    var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
+    var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
+    return nb_items_file1 - nb_items_file2;
   });
-  run(to_run, "./results/add_csv_attachment.csv", function (err) {if (err) throw err;});
+  run(to_run, "../results/add_csv_attachments.csv", function (err) {if (err) throw err;});
 
 });

Milestone1/bench_files/add_csv_metadata_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_csv_metadata.csv", {flags:'a'});
-fs.truncateSync("./results/add_csv_metadata.csv");
+var stream = fs.createWriteStream("../results/add_csv_metadata.csv", {flags:'a'});
+fs.truncateSync("../results/add_csv_metadata.csv");
 stream.write("items,ramUsed,memUsed\n");
 
 var to_run = [];
-fs.readdir("./files/CSV/", function(err, items){
+fs.readdir("../files/CSV/", function(err, items){
   if (err) {
       return console.log('Unable to scan directory: ' + err);
     }
       items.forEach(function (file) {
-        to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "metadata"], amount : parseInt(file.substring(17, file.length-4))});
+        to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "metadata"], amount : parseInt(file.substring(17, file.length-4))});
     });
-    run(to_run, "./results/add_csv_metadata.csv", function (err) {if (err) throw err;});
+    to_run.sort(function(file1, file2){
+      var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
+      var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
+      return nb_items_file1 - nb_items_file2;
+    });
+    run(to_run, "../results/add_csv_metadata.csv", function (err) {if (err) throw err;});
 });

Milestone1/bench_files/add_random_csv_attachments_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_random_csv_attachment.csv", {flags:'a'});
-fs.truncateSync("./results/add_random_csv_attachment.csv");
+var stream = fs.createWriteStream("../results/add_random_csv_attachment.csv", {flags:'a'});
+fs.truncateSync("../results/add_random_csv_attachment.csv");
 stream.write("items,ramUsed,memUsed\n");
 
-var to_run = [{scriptPath : './unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "attachment"], amount : 1000000}];
+var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "attachment"], amount : 1000000}];
 
-run(to_run, "./results/add_random_csv_attachment.csv", function (err) {if (err) throw err;});
+run(to_run, "../results/add_random_csv_attachment.csv", function (err) {if (err) throw err;});

Milestone1/bench_files/add_random_csv_metadata_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_random_csv_metadata.csv", {flags:'a'});
-fs.truncateSync("./results/add_random_csv_metadata.csv");
+var stream = fs.createWriteStream("../results/add_random_csv_metadata.csv", {flags:'a'});
+fs.truncateSync("../results/add_random_csv_metadata.csv");
 stream.write("items,ramUsed,memUsed\n");
 
-var to_run = [{scriptPath : './unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "metadata"], amount : 1000000}];
+var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "metadata"], amount : 1000000}];
 
-run(to_run, "./results/add_random_csv_metadata.csv", function (err) {if (err) throw err;});
+run(to_run, "../results/add_random_csv_metadata.csv", function (err) {if (err) throw err;});

Milestone1/bench_files/run_scripts_sync.js

+function get_size(file){
+  return file.slice(file.indexOf("_")+1, file.lastIndexOf("."));
+}
+
 module.exports = {
   run: function (args, filepath, callback) {
     var ipc=require('node-ipc');
@@ -28,10 +32,12 @@ module.exports = {
       if (mes.includes("DONE")){
         module.exports.run(args, filepath, callback);
       } else if (mes.includes("ERROR")){
-        write(mes);
+        if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
+        else write(mes);
         module.exports.run(args, filepath, callback);
       } else {
-        write(mes);
+        if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
+        else write(mes);
       }
     });
 

Milestone1/results/add_csv_attachments.csv

 items,ramUsed,memUsed
-100,32.16,0
-1000,64.9,11.7
-5000,119.2,61.2
-10000,200.2,143
-50000,401.8,318.5
-100000,720.9,667.5
+100,-0.06638329791223896,2.56640625
+1000,-0.16946998114826495,4.65234375
+5000,-0.30526011830635114,13.390625
+10000,-0.10023595024397591,22.12890625
+50000,-0.003956315988130288,32.82421875
+100000,Timeout ERROR

Milestone1/results/add_csv_metadata.csv

 items,ramUsed,memUsed
-100,28,0
-1000,35.6,0
-5000,111.8,23
-10000,209.6,137.8
-50000,656.2,576.4
-100000,891.6,887.6
+100,-0.03113844575534097,2.13671875
+1000,-0.44977176618101566,3.171875
+5000,-0.42621477611326364,6.37890625
+10000,-0.21620259208293713,10.9140625
+50000,-0.04906697034818297,19.3671875
+100000,Timeout ERROR

Milestone1/results/content_random_size.csv

 items,uncompressed,compressed
+100,0.06,0.04
+1000,0.59,0.41
+5000,2.98,2.07
+10000,5.95,4.14
+50000,29.78,20.66
+100000,59.57,41.3

Milestone1/results/import_index_uncompressed.csv

 items,time
-0.036
+100,0.029
+1000,0.072
+5000,0.193
+10000,0.33
+50000,1.228
+100000,3.604

Milestone1/results/rss.csv

 items,time
-2.892
-38.229
+100,2.861
+1000,28.462
+5000,183.947
+10000,364.987
+50000,2023.26
+100000,4396.794

Milestone1/results/sitemap.csv

 items,time
-6.274
+100,12.671
+1000,136.022
+5000,832.138
+10000,1865.542
+50000,11905.97

Milestone1/unit_tests/add_csv.js

@@ -22,7 +22,8 @@ var saveAs = args.saveAs,
 browser,
 page,
 ramBefore,
-memBefore;
+memBefore,
+time;
 
 function init_server (){
   var port = 9030;
@@ -45,8 +46,19 @@ function init_server (){
   });
 }
 
+function timeout (){ //Every 30 minutes, check if more than 2 hours have passed since last response
+  setTimeout(function () {
+    if (Date.now() - time > 7200000){ //2 hours
+      process.send("Timeout ERROR");
+    } else {
+      timeout();
+    }
+  }, 1800000); //30 minutes
+}
+
 (async () => {
   init_server();
+  timeout();
   browser = await puppeteer.launch();
   page = await browser.newPage();
   os_utils.cpuUsage(function(v){
@@ -55,6 +67,7 @@ function init_server (){
   diskspace.check('C', function (err, result){
     memBefore = result.used/1024/1024;
   });
+  time = Date.now();
   await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/content_load.html?saveAs='+saveAs);
   const [fileChooser] = await Promise.all([
     page.waitForFileChooser(),

Milestone1/unit_tests/add_random_csv.js

@@ -4,8 +4,6 @@ var os_utils 	= require('os-utils');
 const fs = require('fs');
 const puppeteer = require('puppeteer');
 const Server = require('ws').Server;
-var stream = fs.createWriteStream("./results/test.csv", {flags:'a'});
-fs.truncateSync("./results/test.csv");
 const args = require("yargs")
 .usage("Usage : load_content_random.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
 .demandOption(['saveAs'])
@@ -22,6 +20,7 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
   throw 'Unrecognized save as argument';
 }
 var saveAs = args.saveAs,
+stream,
 browser,
 page,
 result,
@@ -54,7 +53,8 @@ function timeout (){ //Every 30 minutes, check if more than 2 hours have passed
   setTimeout(function () {
     if (Date.now() - time > 7200000){ //2 hours
       stream.write(n + ",Timeout error \n");
-      process.exit();
+      result = [];
+      query();
     } else {
       timeout();
     }
@@ -70,6 +70,7 @@ async function query (){
     return;
   } else {
     n += 1000;
+    console.log(n);
     time = Date.now();
     const [fileChooser] = await Promise.all([
       page.waitForFileChooser(),
@@ -80,6 +81,13 @@ async function query (){
 }
 
 (async () => {
+  if (saveAs === "attachment"){
+    stream = fs.createWriteStream("../results/add_random_csv_attachment.csv", {flags:'a'});
+    fs.truncateSync("../results/add_random_csv_attachment.csv");
+  } else {
+    stream = fs.createWriteStream("../results/add_random_csv_metadata.csv", {flags:'a'});
+    fs.truncateSync("../results/add_random_csv_metadata.csv");
+  }
   stream.write("items,ramUsed,memUsed\n");
   init_server();
   result = gen_random_content(args.p);