added some bench scripts

load_content.js → add_csv.js

@@ -32,9 +32,10 @@ function init_server (){
       console.log(msg);
       var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
       var ramUsed = ramAfter - ramBefore;
-      console.log("Ram used : " + ramUsed + " MB");
+      //console.log("Ram used : " + ramUsed + " MB");
       si.mem(function(data){
-        console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
+        process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore));
+        //console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
       });
       browser.close();
     });

add_csv_attachments_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/add_csv_attachments.csv");
+stream.write("items,ramUsed,memUsed\n");
+
+var to_run = [{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100k.csv", "-s", "attachment"], amount : 100000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/50k.csv", "-s", "attachment"], amount : 50000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/10k.csv", "-s", "attachment"], amount : 10000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/5000.csv", "-s", "attachment"], amount : 5000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/1000.csv", "-s", "attachment"], amount : 1000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100.csv", "-s", "attachment"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

add_csv_metadata_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/add_csv_metadata.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/add_csv_metadata.csv");
+stream.write("items,ramUsed,memUsed\n");
+
+var to_run = [{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100k.csv", "-s", "metadata"], amount : 100000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/50k.csv", "-s", "metadata"], amount : 50000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/10k.csv", "-s", "metadata"], amount : 10000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/5000.csv", "-s", "metadata"], amount : 5000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/1000.csv", "-s", "metadata"], amount : 1000},
+              {scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100.csv", "-s", "metadata"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

load_content_random.js → add_random_csv.js

@@ -57,9 +57,10 @@ function init_server (){
       console.log(msg);
       var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
       var ramUsed = ramAfter - ramBefore;
-      console.log("Ram used : " + ramUsed + " MB");
+      //console.log("Ram used : " + ramUsed + " MB");
       si.mem(function(data){
-        console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
+        process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore));
+        //console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
       });
       browser.close();
     });

add_random_csv_attachments_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/add_random_csv_attachment.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/add_random_csv_attachment.csv");
+stream.write("items,ramUsed,memUsed\n");
+
+var to_run = [{scriptPath : './add_random_csv.js', args : ["-p", 100000, "-s", "attachment"], amount : 100000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 50000, "-s", "attachment"], amount : 50000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 10000, "-s", "attachment"], amount : 10000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 5000, "-s", "attachment"], amount : 5000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 1000, "-s", "attachment"], amount : 1000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 100, "-s", "attachment"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

add_random_csv_metadata_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/add_random_csv_metadata.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/add_random_csv_metadata.csv");
+stream.write("items,ramUsed,memUsed\n");
+
+var to_run = [{scriptPath : './add_random_csv.js', args : ["-p", 100000, "-s", "metadata"], amount : 100000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 50000, "-s", "metadata"], amount : 50000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 10000, "-s", "metadata"], amount : 10000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 5000, "-s", "metadata"], amount : 5000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 1000, "-s", "metadata"], amount : 1000},
+              {scriptPath : './add_random_csv.js', args : ["-p", 100, "-s", "metadata"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

add_rss.js

 const puppeteer = require('puppeteer');
-const getPort = require('get-port');
 const Server = require('ws').Server;
 const args = require("yargs")
 .usage("Usage : add_rss.js -file file_path")
@@ -13,11 +12,10 @@ var browser,
 index_start,
 index_total,
 db_start,
-db_total,
-port;
+db_total;
 
 function init_server (){
-  var ws = new Server({port: port});
+  var ws = new Server({port: 9030});
   ws.on('connection', function(w){
     w.on('message', function(msg){
       if (msg === "Index"){
@@ -46,11 +44,10 @@ function init_server (){
 }
 
 (async () => {
-  port = await getPort();
   init_server();
   browser = await puppeteer.launch();
   var page = await browser.newPage();
-  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/add_rss.html?port='+port);
+  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/add_rss.html');
   const [fileChooser] = await Promise.all([
     page.waitForFileChooser(),
     page.click('input#load')

add_sitemap.js

@@ -19,7 +19,6 @@ function init_server (){
       if (msg.includes("Done")){
         var now = Date.now();
         process.send((now-start)/1000);
-        console.log(msg + " in " + (now-start)/1000 +"s");
         ws.close();
         browser.close();
       } else {

import_content.js

@@ -5,10 +5,13 @@ const args = require("yargs")
 .demandOption(['file'])
 .alias("f", "file")
 .alias("u", "uncompressed")
+.aliad("b", "bulk")
 .describe("file", "file containing the index")
 .describe("uncompressed", "to add if file is uncompressed ! For testing only !")
+.describe("bulk", "adds items to db in bulk. Much faster but still in testing")
 .nargs("file", 1)
 .nargs("u", 0)
+.nargs("b", 0)
 .argv;
 
 var browser,
@@ -21,7 +24,8 @@ function init_server (){
     w.on('message', function(msg){
       if (msg.includes("Done")){
           var time_spent = Date.now() - start;
-          console.log(msg + " in " + time_spent/1000 + "s");
+          process.send(time_spent/1000);
+          // console.log(msg + " in " + time_spent/1000 + "s");
           browser.close();
       } else {
         console.log("Error : " + msg);
@@ -39,6 +43,7 @@ function init_server (){
   browser = await puppeteer.launch();
   var page = await browser.newPage();
   if (args.uncompressed) await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content_uncompressed.html');
+  else if (args.bulk) await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content_bulk.html');
   else await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content.html');
   start = Date.now();
   const [fileChooser] = await Promise.all([

import_content_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/import_content.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/import_content.csv");
+stream.write("items,time \n");
+
+var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 100000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 50000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 10000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 5000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 1000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

import_content_bulk_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/import_content_bulk.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/import_content_bulk.csv");
+stream.write("items,time \n");
+
+var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 100000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 50000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 10000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 5000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 1000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

import_content_uncompressed_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/import_content_uncompressed.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/import_content_uncompressed.csv");
+stream.write("items,time \n");
+
+var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 100000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 50000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 10000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 5000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 1000},
+              {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

import_index.js

@@ -21,7 +21,8 @@ function init_server (){
     w.on('message', function(msg){
       if (msg.includes("Done : ")){
           var time_spent = Date.now() - start;
-          console.log(msg + " in " + time_spent/1000 + "s");
+          //console.log(msg + " in " + time_spent/1000 + "s");
+          process.send(time_spent/1000);
           browser.close();
       } else {
         console.log("Error : " + msg);

import_index_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/import_index.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/import_index.csv");
+stream.write("items,time \n");
+
+var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 100000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 50000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 10000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 5000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 1000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

import_index_uncompressed_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/import_index_uncompressed.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/import_index_uncompressed.csv");
+stream.write("items,time \n");
+
+var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 100000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 50000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 10000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 5000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 1000},
+              {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

time_index.js → index_csv.js

+const os = require('os');
 const puppeteer = require('puppeteer');
+const si = require("systeminformation");
 const Server = require('ws').Server;
 const args = require("yargs")
 .usage("Usage : load_index.js -saveAs metadata/attachment -file file_path")
@@ -19,6 +21,8 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
 var saveAs = args.saveAs,
 browser,
 page,
+ramBefore,
+memBefore
 adding_start,
 adding_total,
 saving_start,
@@ -36,9 +40,15 @@ function init_server (){
           saving_start = Date.now();
       } else if (msg === "finished saving"){
           saving_total = Date.now() - saving_start;
-          console.log("Time spent :");
-          console.log("Adding : " + adding_total/1000 + "s");
-          console.log("Saving : " + saving_total/1000 + "s");
+      } else if (msg.includes("done !")) {
+        var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
+        var ramUsed = ramAfter - ramBefore;
+        //console.log("Ram used : " + ramUsed + " MB");
+        si.mem(function(data){
+          //console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
+          process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore) +
+                        "," + adding_total + "," + saving_total);
+        });
         browser.close();
       } else {
         console.log("Error : " + msg);
@@ -55,7 +65,11 @@ function init_server (){
   init_server();
   browser = await puppeteer.launch();
   page = await browser.newPage();
-  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/time_index.html?saveAs='+saveAs);
+  ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
+  si.mem(function(data){
+    memBefore = data.used / 1024 / 1024;
+  });
+  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
   const [fileChooser] = await Promise.all([
     page.waitForFileChooser(),
     page.click('input#load')

time_index_random.js → index_random_csv.js

+const os = require('os');
 const randomstring = require("randomstring");
 const tmp = require('tmp');
 const fs = require('fs');
 const puppeteer = require('puppeteer');
+const si = require("systeminformation");
 const Server = require('ws').Server;
 const args = require("yargs")
 .usage("Usage : load_index.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
@@ -21,6 +23,8 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
 var saveAs = args.saveAs,
 browser,
 page,
+ramBefore,
+memBefore
 adding_start,
 adding_total,
 saving_start,
@@ -61,9 +65,15 @@ function init_server (){
           saving_start = Date.now();
       } else if (msg === "finished saving"){
           saving_total = Date.now() - saving_start;
-          console.log("Time spent :");
-          console.log("Adding : " + adding_total/1000 + "s");
-          console.log("Saving : " + saving_total/1000 + "s");
+      } else if (msg.includes("done !")) {
+        var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
+        var ramUsed = ramAfter - ramBefore;
+        //console.log("Ram used : " + ramUsed + " MB");
+        si.mem(function(data){
+          //console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
+          process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore) +
+                        "," + adding_total + "," + saving_total);
+        });
         browser.close();
       } else {
         console.log("Error : " + msg);
@@ -91,7 +101,11 @@ function gen_random_content (){
   var file_path = gen_random_content();
   browser = await puppeteer.launch();
   page = await browser.newPage();
-  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/time_index.html?saveAs='+saveAs);
+  ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
+  si.mem(function(data){
+    memBefore = data.used / 1024 / 1024;
+  });
+  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
   const [fileChooser] = await Promise.all([
     page.waitForFileChooser(),
     page.click('input#load')

load_index.js

-const os = require('os');
-const puppeteer = require('puppeteer');
-const si = require("systeminformation");
-const Server = require('ws').Server;
-const args = require("yargs")
-.usage("Usage : load_index.js -saveAs metadata/attachment -file file_path")
-.demandOption(['file'])
-.demandOption(['saveAs'])
-.alias("f", "file")
-.alias("s", "saveAs")
-.describe("file", "file containing content to put in the index, data must follow the CSV standard")
-.describe("saveAs", "choose whether to save the index as metadata or attachments in the indexeddb")
-.nargs("file", 1)
-.nargs("saveAs", 1)
-.argv;
-
-if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
-  throw 'Unrecognized save as argument';
-}
-
-var saveAs = args.saveAs,
-browser,
-page,
-ramBefore,
-memBefore;
-
-function init_server (){
-  var port = 9030;
-  var ws = new Server({port: port});
-  ws.on('connection', function(w){
-    w.on('message', function(msg){
-      console.log(msg);
-      var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
-      var ramUsed = ramAfter - ramBefore;
-      console.log("Ram used : " + ramUsed + " MB");
-      si.mem(function(data){
-        console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
-      });
-      browser.close();
-    });
-    w.on('close', function() {
-      ws.close();
-    });
-  });
-}
-
-(async () => {
-  init_server();
-  browser = await puppeteer.launch();
-  page = await browser.newPage();
-  ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
-  si.mem(function(data){
-    memBefore = data.used / 1024 / 1024;
-  });
-  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
-  const [fileChooser] = await Promise.all([
-    page.waitForFileChooser(),
-    page.click('input#load')
-  ]);
-  await fileChooser.accept([args.file]);
-})();

load_index_random.js

-const os = require('os');
-const randomstring = require("randomstring");
-const tmp = require('tmp');
-const fs = require('fs');
-const puppeteer = require('puppeteer');
-const si = require("systeminformation");
-const Server = require('ws').Server;
-const args = require("yargs")
-.usage("Usage : load_index.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
-.demandOption(['saveAs'])
-.demandOption(['paragraphs'])
-.alias("p", "paragraphs")
-.alias("s", "saveAs")
-.describe("paragraphs", "Number of random paragraphs to generate")
-.describe("saveAs", "choose whether to save the index as metadata or attachments in the indexeddb")
-.nargs("saveAs", 1)
-.nargs("paragraphs", 1)
-.argv;
-
-if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
-  throw 'Unrecognized save as argument';
-}
-var saveAs = args.saveAs,
-browser,
-page,
-ramBefore,
-memBefore;
-
-function gen_rdm_string (length){
-  return randomstring.generate({
-    length: length,
-    charset: 'alphabetic'
-  });
-}
-
-function gen_rdm_sentence (word_count, word_length){
-  var tmp_word_array = [];
-  for (var i = 0; i<word_count; i+=1){
-    tmp_word_array[i] = gen_rdm_string(word_length);
-  }
-  return tmp_word_array.join(" ");
-}
-
-function gen_rdm_paragraph (sentence_count, words_per_sentence, word_length){
-  var tmp_sentence_array = [];
-  for (var i=0; i<sentence_count; i+=1){
-    tmp_sentence_array[i] = gen_rdm_sentence(words_per_sentence, word_length);
-  }
-  return tmp_sentence_array.join(". ");
-}
-
-function init_server (){
-  var port = 9030;
-  var ws = new Server({port: port});
-  ws.on('connection', function(w){
-    w.on('message', function(msg){
-      console.log(msg);
-      var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
-      var ramUsed = ramAfter - ramBefore;
-      console.log("Ram used : " + ramUsed + " MB");
-      si.mem(function(data){
-        console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
-      });
-      browser.close();
-    });
-    w.on('close', function() {
-      ws.close();
-    });
-  });
-}
-
-function gen_random_content (){
-  var tmp_para_array = [];
-  for(var i=0; i<args.p; i+=1){
-    tmp_para_array[i] = gen_rdm_paragraph(6, 15, 5);
-  }
-  var tmp_file = tmp.fileSync();
-  fs.writeFileSync(tmp_file.name, tmp_para_array.join("\n"));
-  return tmp_file.name;
-}
-
-(async () => {
-  init_server();
-  var file_path = gen_random_content();
-  browser = await puppeteer.launch();
-  page = await browser.newPage();
-  ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
-  si.mem(function(data){
-    memBefore = data.used / 1024 / 1024;
-  });
-  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
-  const [fileChooser] = await Promise.all([
-    page.waitForFileChooser(),
-    page.click('input#load')
-  ]);
-  await fileChooser.accept([file_path]);
-})();

results/sitemap.csv

+items,time 
+135,8.412

rss_bench.js

@@ -48,6 +48,7 @@ function runScriptSync(list, callback) {
     });
 }
 
+fs.truncateSync("./results/rss.csv");
 stream.write("items,time \n");
 
 var to_run = [{scriptPath : './add_rss.js', args : ['-f' ,"./test_files/RSS/rss_100000.xml"], amount : 100000},

sitemap_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/sitemap.csv", {flags:'a'});
+
+function writeResult(items, time){
+  stream.write(items + "," + time + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/sitemap.csv");
+stream.write("items,time \n");
+
+// var to_run = [{scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 100000},
+//               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 50000},
+//               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 10000},
+//               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 5000},
+//               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 1000},
+//               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 100}];
+
+var to_run = [{scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/44svt_135.xml"], amount : 135}];
+runScriptSync(to_run, function (err) {if (err) throw err;});

small

-1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20