added last bench scripts, all results

download_content.js

@@ -7,7 +7,31 @@ const args = require("yargs")
 .nargs("file", 1)
 .argv;
 
+function init_server (){
+  var port = 9030;
+  var ws = new Server({port: port});
+  ws.on('connection', function(w){
+    w.on('message', function(msg){
+      if (msg === "Start"){
+          start = Date.now();
+      } else if (msg === "Done"){
+          var total = Date.now() - start;
+          //console.log("Time spent : " + total/1000 + "s");
+          process.send(total/1000);
+          // NEED TO SEND SIZE
+      } else {
+        console.log("Error : " + msg);
+        browser.close();
+      }
+    });
+    w.on('close', function() {
+      ws.close();
+    });
+  });
+}
+
 (async () => {
+  init_server();
   var browser = await puppeteer.launch({headless: false});
   var page = await browser.newPage();
   await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/download_content.html');

download_content_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/content_csv_size.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/content_csv_size.csv");
+stream.write("items,uncompressed,compressed\n");
+
+var to_run = [{scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/100k.csv"], amount : 100000},
+              {scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/50k.csv"], amount : 50000},
+              {scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/10k.csv"], amount : 10000},
+              {scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/5000.csv"], amount : 5000},
+              {scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/1000.csv"], amount : 1000},
+              {scriptPath : './download_content.js', args : ['-f' ,"./test_files/CSV/100.csv"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

download_content_random.js

@@ -22,7 +22,9 @@ function init_server (){
         start = Date.now();
       } else if (msg.includes("Done")){
           var time_spent = Date.now() - start;
-          console.log(msg + " in " + time_spent/1000 + "s");
+          //console.log(msg + " in " + time_spent/1000 + "s");
+          process.send(time_spent/1000);
+          // NEED TO SEND SIZE
       } else {
         console.log("Error : " + msg);
         browser.close();

download_content_random_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/content_random_size.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/content_random_size.csv");
+stream.write("items,uncompressed,compressed\n");
+
+var to_run = [{scriptPath : './download_content_random.js', args : ['-p' ,"100000"], amount : 100000},
+              {scriptPath : './download_content_random.js', args : ['-p' ,"50000"], amount : 50000},
+              {scriptPath : './download_content_random.js', args : ['-p' ,"10000"], amount : 10000},
+              {scriptPath : './download_content_random.js', args : ['-p' ,"5000"], amount : 5000},
+              {scriptPath : './download_content_random.js', args : ['-p' ,"1000"], amount : 1000},
+              {scriptPath : './download_content_random.js', args : ['-p' ,"100"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

download_index.js

@@ -20,7 +20,9 @@ function init_server (){
           start = Date.now();
       } else if (msg === "Done"){
           var total = Date.now() - start;
-          console.log("Time spent : " + total/1000 + "s");
+          //console.log("Time spent : " + total/1000 + "s");
+          process.send(total/1000);
+          // NEED TO SEND SIZE 
       } else {
         console.log("Error : " + msg);
         browser.close();

download_index_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/index_csv_size.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/index_csv_size.csv");
+stream.write("items,uncompressed,compressed\n");
+
+var to_run = [{scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/100k.csv"], amount : 100000},
+              {scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/50k.csv"], amount : 50000},
+              {scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/10k.csv"], amount : 10000},
+              {scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/5000.csv"], amount : 5000},
+              {scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/1000.csv"], amount : 1000},
+              {scriptPath : './download_index.js', args : ['-f' ,"./test_files/CSV/100.csv"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

download_index_random.js

@@ -8,7 +8,7 @@ const args = require("yargs")
 .demandOption(['paragraphs'])
 .alias("p", "paragraphs")
 .describe("paragraphs", "Number of random paragraphs to generate")
-.nargs("prom", 1)
+.nargs("p", 1)
 .argv;
 
 var browser,
@@ -23,7 +23,9 @@ function init_server (){
           start = Date.now();
       } else if (msg === "Done"){
           var total = Date.now() - start;
-          console.log("Time spent : " + total/1000 + "s");
+          //console.log("Time spent : " + total/1000 + "s");
+          process.send(total/1000);
+          // NEED TO SEND SIZE
       } else {
         console.log("Error : " + msg);
         browser.close();

download_index_random_bench.js

+var childProcess = require('child_process');
+const fs = require('fs');
+var stream = fs.createWriteStream("./results/index_random_size.csv", {flags:'a'});
+
+function writeResult(items, info){
+  stream.write(items + "," + info + "\n");
+}
+
+function runScriptSync(list, callback) {
+    if (list.length === 0) return;
+    var invoked = false,
+        to_run = list.pop();
+        process = childProcess.fork(to_run.scriptPath, to_run.args);
+    process.on('message', function (mes){
+      console.log(to_run.amount + " : " + mes)
+      writeResult(to_run.amount, mes);
+      runScriptSync(list, callback);
+    });
+    process.on('error', function (err) {
+        if (invoked) return;
+        invoked = true;
+        callback(err);
+    });
+    process.on('exit', function (code) {
+        if (invoked) return;
+        invoked = true;
+        var err = code === 0 ? null : new Error('exit code ' + code);
+        callback(err);
+    });
+}
+
+fs.truncateSync("./results/index_random_size.csv");
+stream.write("items,uncompressed,compressed\n");
+
+var to_run = [{scriptPath : './download_index_random.js', args : ['-p' ,"100000"], amount : 100000},
+              {scriptPath : './download_index_random.js', args : ['-p' ,"50000"], amount : 50000},
+              {scriptPath : './download_index_random.js', args : ['-p' ,"10000"], amount : 10000},
+              {scriptPath : './download_index_random.js', args : ['-p' ,"5000"], amount : 5000},
+              {scriptPath : './download_index_random.js', args : ['-p' ,"1000"], amount : 1000},
+              {scriptPath : './download_index_random.js', args : ['-p' ,"100"], amount : 100}];
+
+runScriptSync(to_run, function (err) {if (err) throw err;});

import_content_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/import_content.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content100000.zip"], amount : 100000},
               {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content10000.zip"], amount : 10000},

import_content_bulk_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/import_content_bulk.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content100000.zip", '-b'], amount : 100000},
               {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content10000.zip", '-b'], amount : 10000},

import_content_uncompressed_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/import_content_uncompressed.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content100000.json", '-u'], amount : 100000},
               {scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/content10000.json", '-u'], amount : 10000},

import_index_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/import_index.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/index100000.zip"], amount : 100000},
               {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/index50000.zip"], amount : 50000},

import_index_uncompressed_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/import_index_uncompressed.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/index100000.json", '-u'], amount : 100000},
               {scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/index50000.json", '-u'], amount : 50000},

results/add_csv_attachments.csv

+items,ramUsed,memUsed
+100,32.16,0
+1000,64.9,11.7
+5000,119.2,61.2
+10000,200.2,143
+50000,401.8,318.5
+100000,720.9,667.5

results/add_csv_metadata.csv

+items,ramUsed,memUsed
+100,28,0
+1000,35.6,0
+5000,111.8,23
+10000,209.6,137.8
+50000,656.2,576.4
+100000,891.6,887.6

results/add_random_csv_attachment.csv

+items,ramUsed,memUsed
+100,53.3,0
+1000,102,0
+5000,203.6,0
+10000,318.2,75.9
+50000,955.9,521.8
+100000,1532.1,1258.8

results/add_random_csv_metadata.csv

+items,ramUsed,memUsed
+100,41.1,0
+1000,101.6,0
+5000,220,0
+10000,408.9,172.1

results/content_random_size.csv

+items,uncompressed,compressed
+100,0.06,0.04
+1000,0.59,0.41
+5000,2.98,2.07
+10000,5.95,4.14
+50000,29.78,20.66
+100000,59.57,41.3

results/content_size.csv

+items,uncompressed,compressed
+100,0.01,0.008
+1000,0.18,0.06
+5000,0.93,0.32
+10000,1.86,0.63
+50000,9.31,3.17
+100000,18.63,6.32

results/import_content_bulk.csv

-items,time 
+items,time
 100,0.637
 1000,1.018
 10000,3.993

results/import_content_uncompressed.csv

-items,time 
+items,time
 100,3.445
 1000,34.34
 10000,353.906

results/import_index.csv

-items,time 
+items,time
 100,0.099
 1000,0.497
 5000,2.797

results/import_index_uncompressed.csv

-items,time 
+items,time
 100,0.029
 1000,0.072
 5000,0.193

results/index_csv_size.csv

+items,uncompressed,compressed
+100,0.27,0.01
+1000,2.65,0.27
+5000,13.19,2.03
+10000,26.3,4.26
+50000,130.76,21.67
+100000,260.16,43.25

results/index_random_size.csv

+items,uncompressed,compressed
+100,0.4,0.04
+1000,3.72,0.97
+5000,18.07,7.78
+10000,35.71,16.95
+50000,175.06,94.33
+100000,348.55,191.22

results/rss.csv

@@ -4,3 +4,4 @@ items,time
 5000,183.947
 10000,364.987
 50000,2023.26
+100000,4396.794

results/sitemap.csv

+items,time
+100,12.671
+1000,136.022
+5000,832.138
+10000,1865.542
+50000,11905.97

sitemap_bench.js

@@ -30,7 +30,7 @@ function runScriptSync(list, callback) {
 }
 
 fs.truncateSync("./results/sitemap.csv");
-stream.write("items,time \n");
+stream.write("items,time\n");
 
 var to_run = [{scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/sitemap50000.xml"], amount : 50000},
               {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/sitemap10000.xml"], amount : 10000},