final rerun of all tests

Milestone1/bench_files/add_csv_attachments_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
-fs.truncateSync("./results/add_csv_attachments.csv");
+
+var stream = fs.createWriteStream("../results/add_csv_attachments.csv", {flags:'a'});
+fs.truncateSync("../results/add_csv_attachments.csv");
 stream.write("items,ramUsed,memUsed\n");
 
 var to_run = [];
-fs.readdir("./files/CSV/", function(err, items){
+fs.readdir("../files/CSV/", function(err, items){
   if (err) {
     return console.log('Unable to scan directory: ' + err);
   }
   items.forEach(function (file) {
-    to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "attachment"], amount : parseInt(file.substring(17, file.length-4))});
+    to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "attachment"]});
+  });
+  to_run.sort(function(file1, file2){
+    var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
+    var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
+    return nb_items_file1 - nb_items_file2;
   });
-  run(to_run, "./results/add_csv_attachment.csv", function (err) {if (err) throw err;});
+  run(to_run, "../results/add_csv_attachments.csv", function (err) {if (err) throw err;});
 
 });

Milestone1/bench_files/add_csv_metadata_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_csv_metadata.csv", {flags:'a'});
-fs.truncateSync("./results/add_csv_metadata.csv");
+var stream = fs.createWriteStream("../results/add_csv_metadata.csv", {flags:'a'});
+fs.truncateSync("../results/add_csv_metadata.csv");
 stream.write("items,ramUsed,memUsed\n");
 
 var to_run = [];
-fs.readdir("./files/CSV/", function(err, items){
+fs.readdir("../files/CSV/", function(err, items){
   if (err) {
       return console.log('Unable to scan directory: ' + err);
     }
       items.forEach(function (file) {
-        to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "metadata"], amount : parseInt(file.substring(17, file.length-4))});
+        to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "metadata"], amount : parseInt(file.substring(17, file.length-4))});
     });
-    run(to_run, "./results/add_csv_metadata.csv", function (err) {if (err) throw err;});
+    to_run.sort(function(file1, file2){
+      var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
+      var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
+      return nb_items_file1 - nb_items_file2;
+    });
+    run(to_run, "../results/add_csv_metadata.csv", function (err) {if (err) throw err;});
 });

Milestone1/bench_files/add_random_csv_attachments_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_random_csv_attachment.csv", {flags:'a'});
-fs.truncateSync("./results/add_random_csv_attachment.csv");
+var stream = fs.createWriteStream("../results/add_random_csv_attachment.csv", {flags:'a'});
+fs.truncateSync("../results/add_random_csv_attachment.csv");
 stream.write("items,ramUsed,memUsed\n");
 
-var to_run = [{scriptPath : './unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "attachment"], amount : 1000000}];
+var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "attachment"], amount : 1000000}];
 
-run(to_run, "./results/add_random_csv_attachment.csv", function (err) {if (err) throw err;});
+run(to_run, "../results/add_random_csv_attachment.csv", function (err) {if (err) throw err;});

Milestone1/bench_files/add_random_csv_metadata_bench.js

 var run = require('./run_scripts_sync').run;
 const fs = require('fs');
-var stream = fs.createWriteStream("./results/add_random_csv_metadata.csv", {flags:'a'});
-fs.truncateSync("./results/add_random_csv_metadata.csv");
+var stream = fs.createWriteStream("../results/add_random_csv_metadata.csv", {flags:'a'});
+fs.truncateSync("../results/add_random_csv_metadata.csv");
 stream.write("items,ramUsed,memUsed\n");
 
-var to_run = [{scriptPath : './unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "metadata"], amount : 1000000}];
+var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "metadata"], amount : 1000000}];
 
-run(to_run, "./results/add_random_csv_metadata.csv", function (err) {if (err) throw err;});
+run(to_run, "../results/add_random_csv_metadata.csv", function (err) {if (err) throw err;});

Milestone1/bench_files/run_scripts_sync.js

+function get_size(file){
+  return file.slice(file.indexOf("_")+1, file.lastIndexOf("."));
+}
+
 module.exports = {
   run: function (args, filepath, callback) {
     var ipc=require('node-ipc');
@@ -28,10 +32,12 @@ module.exports = {
       if (mes.includes("DONE")){
         module.exports.run(args, filepath, callback);
       } else if (mes.includes("ERROR")){
-        write(mes);
+        if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
+        else write(mes);
         module.exports.run(args, filepath, callback);
       } else {
-        write(mes);
+        if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
+        else write(mes);
       }
     });
 

Milestone1/results/add_csv_attachments.csv

-items,ramUsed,memUsed
-100,32.16,0
-1000,64.9,11.7
-5000,119.2,61.2
-10000,200.2,143
-50000,401.8,318.5
-100000,720.9,667.5
+items,ramUsed,memUsed
+100,-0.06638329791223896,2.56640625
+1000,-0.16946998114826495,4.65234375
+5000,-0.30526011830635114,13.390625
+10000,-0.10023595024397591,22.12890625
+50000,-0.003956315988130288,32.82421875
+100000,Timeout ERROR

Milestone1/results/add_csv_metadata.csv

-items,ramUsed,memUsed
-100,28,0
-1000,35.6,0
-5000,111.8,23
-10000,209.6,137.8
-50000,656.2,576.4
-100000,891.6,887.6
+items,ramUsed,memUsed
+100,-0.03113844575534097,2.13671875
+1000,-0.44977176618101566,3.171875
+5000,-0.42621477611326364,6.37890625
+10000,-0.21620259208293713,10.9140625
+50000,-0.04906697034818297,19.3671875
+100000,Timeout ERROR

Milestone1/results/add_random_csv_attachment.csv

 items,ramUsed,memUsed
-1000,49.30078125,96.55078125
-2000,178.94140625,-17.5
-3000,106.48828125,80.24609375
-4000,150.515625,10.12109375
-5000,152.6015625,-4.84765625
-6000,168.3515625,26.34765625
-7000,108.5234375,25.72265625
-8000,174.24609375,-28.44921875
-9000,245.22265625,-110.80859375
-10000,307.796875,-157.94140625
-11000,292.6328125,89.296875
-12000,268.45703125,-121.4921875
-13000,172.484375,-6.5390625
-14000,300.78515625,81.37109375
-15000,293.015625,-167.6875
-16000,355.36328125,-245.0390625
-17000,254.93359375,-165.453125
-18000,417.12890625,-296.078125
-19000,416.07421875,51.3515625
-20000,392.8828125,48.1640625
-21000,207.35546875,-113.38671875
-22000,322.1875,50.01171875
-23000,487.609375,-274.7109375
-24000,413.46875,100.234375
-25000,616.59765625,-420.63671875
-26000,480.8984375,59.05078125
-27000,480.05078125,51.05078125
-28000,514.12890625,74.0703125
-29000,655.87890625,-542.4921875
-30000,537.5859375,-460.06640625
+1000,-0.08832028900884736,5.18359375
+2000,-0.17942280845802006,7.19921875
+3000,-0.1755468242060515,12.03125
+4000,-0.18342907623350002,15.47265625
+5000,-0.17194676804730946,10.51953125
+6000,-0.17943754276892576,21.37890625
+7000,-0.18729682420605154,23.6328125
+8000,-0.18329732414355937,27.13671875
+9000,-0.18529682420605154,29.1328125
+10000,-0.18532735104012343,32.65625
+11000,-0.18729682420605154,35.01171875
+12000,-0.17342702538061106,38.3828125
+13000,-0.1795468242060515,21.60546875
+14000,-0.18342133977049713,44.21875
+15000,-0.18342133977049713,46.1484375
+16000,-0.18167041760440117,27.828125
+17000,-0.18329732414355937,27.375
+18000,-0.181432819215409,33.82421875
+19000,-0.18129907336261786,36.12890625
+20000,-0.18729682420605154,39.69140625
+21000,-0.18542182420605158,63.62109375
+22000,-0.18729682420605154,45.6796875
+23000,-0.15604291746770926,65.625
+24000,-0.18142182420605157,51.4453125
+25000,-0.15193508485830698,67.98828125
+26000,-0.18729682420605154,57.19921875
+27000,-0.15991840190326367,53.3125
+28000,-0.18354588647161796,62.875
+29000,-0.16979682420605158,62.41796875
+30000,-0.17380816694057022,68.625
+31000,-0.12478901072936699,52.13671875
+32000,-0.18529682420605154,52.90625
+33000,-0.17571669719820693,51.7421875
+34000,-0.181432819215409,58.93359375
+35000,-0.18529682420605154,53.53125
+36000,-0.18529682420605154,76.7421875
+37000,-0.18729682420605154,79.37890625
+38000,-0.18542158980175105,79.59765625
+39000,-0.18142182420605157,61.23828125
+40000,-0.1559335854205961,61.67578125
+41000,-0.18529707417480545,64.19921875
+42000,-0.18529682420605154,85.50390625
+43000,-0.17744276408381132,66.88671875
+44000,-0.18143355075295775,67.88671875
+45000,-0.18142182420605157,70.359375
+46000,-0.17544700507171196,96.66796875
+47000,-0.18342133977049713,73.20703125
+48000,-0.17369732327030607,318.78125
+49000,-0.18342182420605158,306.2578125
+50000,-0.17557174000996822,342.15234375
+51000,-0.1775456053036888,302.96875
+52000,-0.16004682420605154,303.32421875
+53000,-0.17967182420605154,327.00390625
+54000,-0.14437729831709334,331.2265625
+55000,-0.18729682420605154,329.86328125
+56000,-0.18529682420605154,309.1484375
+57000,-0.18729682420605154,332.7421875
+58000,-0.16010017148638622,348.109375
+59000,-0.18729682420605154,314.46484375
+60000,-0.1422968242060515,347.56640625
+61000,-0.18330480823798767,317.3359375
+62000,-0.11141233864535649,320.953125
+63000,-0.18729682420605154,341.14453125
+64000,-0.16391890196577152,330.58203125
+65000,-0.1561679330946626,344.6640625
+66000,-0.11092182420605157,340.625
+67000,-0.17943950317836388,338.109375
+68000,0.056013692284303196,334.66796875
+69000,-0.1735502608468913,351.53515625
+70000,-0.1796699174793699,332.37109375
+71000,-0.16591915199702545,354.83984375
+72000,-0.1756921324166829,334.30078125
+73000,-0.17367182420605154,358.04296875
+74000,-0.15428444456368562,360.70703125
+75000,-0.18330530618185292,361.69140625
+76000,-0.18143428210800983,362.1640625
+77000,-0.1735468242060515,360.25390625
+78000,-0.18142329258895584,365.44921875
+79000,-0.16612357722635296,383.171875
+80000,0.12726941989039386,368.5703125
+81000,0.01929000214125387,376.8984375
+82000,-0.05443843150513916,372.24609375
+83000,0.13213895160221212,377.078125
+84000,0.0780143894648977,375.43359375
+85000,0.042867846452631087,357.73828125
+86000,-0.03177447331575145,358.46875
+87000,0.11364750978997229,359.0234375
+88000,-0.056530478412827434,361.52734375
+89000,-0.050836801753762706,376.85546875
+90000,-0.181432819215409,386.17578125
+91000,-0.17943754276892576,364.7734375
+92000,-0.18529682420605154,388.12890625
+93000,-0.1579804569405825,390.8515625
+94000,-0.18542182420605158,390.71484375
+95000,-0.1814210897392432,392.42578125
+96000,-0.1812983238311453,373.578125
+97000,-0.14039063658130102,407.51953125
+98000,-0.18542158980175105,377.390625
+99000,-0.16379976133890994,416.09375
+100000,-0.18529682420605154,400.58984375
+101000,-0.1675492926474964,419.05859375
+102000,-0.17943852309591057,383.25
+103000,-0.15404266743645534,419.91796875
+104000,-0.1814210897392432,406.84765625
+105000,-0.12279682420605154,416.828125
+106000,-0.1812983238311453,409.828125
+107000,-0.17166987083688034,410.47265625
+108000,-0.17746239124202812,394.734375
+109000,-0.1756703708993882,407.5
+110000,-0.18342230852051222,399.7109375
+111000,-0.18130880025395568,397.41796875
+112000,-0.17369732327030607,439.453125
+113000,-0.18354588647161796,399.3203125
+114000,-0.18542158980175105,422.26953125
+115000,-0.15980369748773116,424.51953125
+116000,-0.17161347842561836,413.953125
+117000,-0.18729682420605154,427.41796875
+118000,-0.17357769225294606,428.3203125
+119000,-0.18342182420605158,430.93359375
+120000,-0.14653153722343304,440.28515625
+121000,-0.17756749902206748,433.69140625
+122000,-0.17170301183080205,428.6484375
+123000,-0.17731927369470213,437.0703125
+124000,-0.1814210897392432,431.02734375
+125000,-0.18529682420605154,419.0234375
+126000,-0.17379344836209054,419.48828125
+127000,-0.1795458553349426,442.49609375
+128000,-0.15967872872992062,452.9296875
+129000,-0.18154610536619653,445.265625
+130000,-0.13078269567391854,453.3515625
+131000,-0.17575033114623084,449.26953125
+132000,-0.18342182420605158,429.265625
+133000,-0.17744030517922682,452.5859375
+134000,-0.12303428365202107,462.4296875
+135000,-0.1795477928349729,455.5078125
+136000,-0.16204682420605154,474.53515625
+137000,-0.18342133977049713,458.4375
+138000,-0.10906638181664885,461.42578125
+139000,-0.17744153478489388,462.26171875
+140000,-0.1421484081421952,478.734375
+141000,-0.1657149878787062,472.84765625
+142000,-0.09338313001797471,471.37109375
+143000,-0.18130880025395568,468.140625
+144000,-0.13452077929026862,488.28125
+145000,-0.18542158980175105,471.265625
+146000,-0.13254682420605157,470.0625
+147000,-0.16179044761190298,487.61328125
+148000,-0.1286864767929048,487.35546875
+149000,-0.18729682420605154,456.609375
+150000,-0.09575105131960837,483.25390625
+151000,-0.12856884719742978,484.7734375
+152000,-0.13654682420605158,468.56640625
+153000,-0.14841210302575647,493.85546875
+154000,-0.16191547886971747,469.3671875
+155000,-0.16767427702444926,505.03515625
+156000,-0.18142182420605157,467.796875
+157000,-0.1815468242060515,502.8671875
+158000,-0.1677992614014021,472.375
+159000,-0.17367182420605154,498.0859375
+160000,-0.15604291746770926,494.5234375
+161000,-0.15785937877758704,500.46875
+162000,-0.18729682420605154,490.08984375
+163000,-0.16440169865213716,503.171875
+164000,-0.18142255848926614,472.80859375
+165000,-0.1480615374385893,487.08984375
+166000,-0.1815468242060515,475.7265625
+167000,-0.18542158980175105,493.1640625
+168000,-0.1600434175302171,507.08203125
+169000,-0.17544700507171196,479.5
+170000,-0.18138913189835926,502.546875
+171000,-0.18529682420605154,482.421875
+172000,-0.17943754276892576,485.4453125
+173000,-0.1756674631956726,527.34765625
+174000,-0.16995217733807055,524.11328125
+175000,-0.1737951364950877,506.1484375
+176000,-0.1815468242060515,512.19921875
+177000,-0.11544053678090183,498.96875
+178000,-0.18529682420605154,494.34375
+179000,-0.09553535402227853,520.72265625
+180000,-0.18729682420605154,518.15625
+181000,-0.17167182420605154,540.76953125
+182000,-0.1662506703598977,538.77734375
+183000,-0.18342182420605158,542.4765625
+184000,-0.15416768306340867,505.41015625
+185000,-0.1599218242060515,542.01171875
+186000,-0.1247811952988247,538.6015625
+187000,-0.12879682420605154,541.66015625
+188000,-0.16591915199702545,543.96875
+189000,-0.08602150537634412,534.16015625
+190000,-0.18729682420605154,513.078125
+191000,-0.18154610536619653,557.203125
+192000,-0.15205122990533915,547.12109375
+193000,-0.16967182420605154,557.3828125
+194000,-0.10918158861050098,541.03515625
+195000,-0.05618821278247943,555.39453125
+196000,-0.15393016587063535,553.69140625
+197000,-0.18130954716975367,523.0078125
+198000,-0.14031444259915415,553.2109375
+199000,-0.07981026002657399,550.1328125
+200000,-0.12303428365202107,552.1796875
+201000,-0.10095420268912036,545.1875
+202000,-0.1306789014463965,556.46875
+203000,-0.04907327131183992,534.68359375
+204000,-0.18729682420605154,555.75390625
+205000,-0.1814210897392432,537.09765625
+206000,-0.16180001130766386,576.59375
+207000,-0.1796699174793699,538.63671875
+208000,-0.12467965135265824,566.0703125
+209000,-0.15640831817602452,569.30078125
+210000,-0.1639218242060515,559.4453125
+211000,-0.18329682420605153,563.35546875
+212000,-0.18542158980175105,547.39453125
+213000,-0.16766691672918232,583.078125
+214000,-0.13466524525868306,576.734375
+215000,-0.18729682420605154,552.796875
+216000,-0.1365531671631819,583.70703125
+217000,-0.18329682420605153,576.3046875
+218000,-0.1482968242060515,594.3671875
+219000,-0.18729682420605154,579.5703125
+220000,-0.11857259579170687,586.3359375
+221000,-0.18529707417480545,577.203125
+222000,-0.14705067035989772,578.76171875
+223000,-0.18529682420605154,586.8984375
+224000,-0.1306789014463965,597.38671875
+225000,-0.18729682420605154,589.01953125
+226000,-0.08565911949296245,568.75390625
+227000,-0.18342182420605158,591.89453125
+228000,-0.09941083845783305,599.80859375
+229000,-0.18168165075958287,580.75390625
+230000,-0.1559218242060515,612.1875
+231000,-0.13453363340835212,609.93359375
+232000,-0.1307848481581474,595.0546875
+233000,-0.18542158980175105,594.55078125
+234000,-0.13466524525868306,612.33984375
+235000,-0.1814210897392432,598.578125
+236000,-0.05069502779886592,610.0625
+237000,-0.18529682420605154,601.91015625
+238000,-0.16404101025256312,626.78125
+239000,-0.15616014795245647,619.43359375
+240000,-0.05622097764851064,622.2265625
+241000,-0.15592574559087846,622.05859375
+242000,-0.07981026002657399,618.19140625
+243000,-0.08600884121242891,621.6875
+244000,-0.05451362840710183,623.91015625
+245000,-0.13464554535389928,622.953125
+246000,-0.04656441575710768,637.33203125
+247000,-0.1323174414745758,614.0390625
+248000,-0.17342182420605157,618.984375
+249000,-0.1226718242060515,632.74609375
+250000,-0.05242182420605157,635.33984375
+251000,-0.15992524565337063,646.546875
+252000,-0.050581003161312754,635.4453125
+253000,-0.15036146109150195,628.49609375
+254000,-0.050598086232791495,622.1875
+255000,-0.15404682420605154,631.05859375
+256000,-0.18729682420605154,617.65625
+257000,Timeout error

Milestone1/results/add_random_csv_metadata.csv

-items,ramUsed,memUsed
-100,41.1,0
-1000,101.6,0
-5000,220,0
-10000,408.9,172.1
+items,ramUsed,memUsed
+1000,-0.1729192546583851,3.6328125
+2000,-0.1514232308015262,4.01171875
+3000,-0.16507465249178066,5.6796875
+4000,-0.1729192546583851,6.00390625
+5000,-0.16518375871328217,10.77734375
+6000,-0.15907367651194626,7.23828125
+7000,-0.16323788210936552,13.61328125
+8000,-0.17094929651999557,8.42578125
+9000,-0.0773603724228561,10.11328125
+10000,-0.1709192546583851,17.93359375
+11000,-0.1710440202540846,11.9765625
+12000,-0.170949539036078,21.78515625
+13000,-0.17104425465838513,23.39453125
+14000,-0.1671800157313732,24.9609375
+15000,-0.16316803575602234,26.625
+16000,-0.15747908531459232,27.09375
+17000,-0.16904377022283068,16.73828125
+18000,-0.16704425465838513,31.37890625
+19000,-0.1729192546583851,32.94140625
+20000,-0.16516731667388895,34.52734375
+21000,-0.16507465249178066,36.21484375
+22000,-0.1729192546583851,19.96875
+23000,-0.16904425465838513,21.53515625
+24000,-0.1709192546583851,21.5625
+25000,-0.1709192546583851,23.1953125
+26000,-0.1689192546583851,44.109375
+27000,-0.14954425465838506,45.734375
+28000,-0.16704425465838513,24.8359375
+29000,-0.16904425465838513,26.3984375
+30000,-0.16518375871328217,28.22265625
+31000,-0.1710440202540846,29.78515625
+32000,-0.1729192546583851,49.93359375
+33000,-0.1709192546583851,51.55859375
+34000,-0.170919504627139,29.75
+35000,-0.16917580569407265,31.375
+36000,-0.1710440202540846,52.97265625
+37000,-0.16104573884786144,54.6171875
+38000,-0.15329915968212915,32.953125
+39000,-0.1729192546583851,34.578125
+40000,-0.1709192546583851,55.84765625
+41000,-0.16917627274946434,57.37890625
+42000,-0.16493522272225736,57.3515625
+43000,-0.13003889461337947,58.9765625
+44000,-0.1729192546583851,58.93359375
+45000,-0.1652942546583851,60.5
+46000,-0.1709192546583851,60.46875
+47000,-0.16692000456464684,62.15625
+48000,-0.170919504627139,40.703125
+49000,-0.16892723869032122,42.265625
+50000,-0.16330986990860352,64.16796875
+51000,-0.1673033804539673,65.85546875
+52000,-0.16904425465838513,65.7421875
+53000,-0.17104425465838513,67.38671875
+54000,-0.1729192546583851,67.390625
+55000,-0.16691925465838509,68.953125
+56000,-0.16716853581853008,68.94921875
+57000,-0.1709192546583851,49.328125
+58000,-0.16704425465838513,49.33984375
+59000,-0.1689202544084476,50.96484375
+60000,-0.16910340039939942,72.13671875
+61000,-0.16905150668583357,73.69921875
+62000,-0.16716853581853008,73.81640625
+63000,-0.16505997322125932,75.56640625
+64000,-0.16119270775219752,54.2890625
+65000,-0.16716781679892023,55.8984375
+66000,-0.1729192546583851,77.58984375
+67000,-0.16506095354824413,79.21875
+68000,-0.16121606740738903,57.546875
+69000,-0.1709192546583851,59.171875
+70000,-0.1476946945501041,80.8203125
+71000,-0.16705671256034338,82.3828125
+72000,-0.15410205035730984,81.90625
+73000,-0.14754108238685115,83.53125
+74000,-0.1729192546583851,62.515625
+75000,-0.16904425465838513,64.078125
+76000,-0.1729192546583851,85.58984375
+77000,-0.1729192546583851,87.21484375
+78000,-0.16891975459589292,87.27734375
+79000,-0.16891975459589292,88.90234375
+80000,-0.16917580569407265,88.83984375
+81000,-0.1729192546583851,90.40625
+82000,-0.16506095354824413,90.37890625
+83000,-0.1729192546583851,92.0078125
+84000,-0.16904377022283068,70.5703125
+85000,-0.16705524966774254,72.1953125
+86000,-0.16717929957977262,93.5390625
+87000,-0.16306396523722744,95.16015625
+88000,-0.1670449889415997,95.1484375
+89000,-0.1475949033609799,96.8515625
+90000,-0.1729192546583851,75.48828125
+91000,-0.1691683169239515,77.05078125
+92000,-0.16904377022283068,98.4453125
+93000,-0.1729192546583851,100.0703125
+94000,-0.1709192546583851,78.69140625
+95000,-0.16905150668583357,79.0625
+96000,-0.1709192546583851,101.625
+97000,-0.16904425465838513,82.39453125
+98000,-0.16530852477691282,103.625
+99000,-0.11828059499346888,86.69140625
+100000,-0.16304548887910753,83.59375
+101000,-0.12003764445710996,92.12109375
+102000,-0.1729192546583851,85.15625
+103000,-0.12616925465838513,97.88671875
+104000,-0.1729192546583851,108.0546875
+105000,-0.16129280135172175,106.63671875
+106000,-0.16717929957977262,109.7109375
+107000,-0.1575404099472073,108.38671875
+108000,-0.15001849129960643,90.86328125
+109000,-0.16904377022283068,90.21484375
+110000,-0.1709192546583851,94.16015625
+111000,-0.16704425465838513,113.73828125
+112000,-0.1729192546583851,93.32421875
+113000,-0.16705671256034338,115.859375
+114000,-0.1709192546583851,116.5
+115000,-0.16904425465838513,100.4609375
+116000,-0.16505997322125932,96.578125
+117000,-0.15353940969714475,103.78515625
+118000,-0.1611912446771,98.15234375
+119000,-0.170919504627139,98.66796875
+120000,-0.16904425465838513,121.42578125
+121000,-0.16504136199467268,121.22265625
+122000,-0.1729192546583851,123.46875
+123000,-0.16519820858117462,101.99609375
+124000,-0.16704425465838513,124.6796875
+125000,-0.16516828578727616,124.41015625
+126000,-0.16917580569407265,126.78515625
+127000,-0.1709192546583851,126.12109375
+128000,-0.1729192546583851,106.91796875
+129000,-0.16705524966774254,107.078125
+130000,-0.16910246076525537,112.64453125
+131000,-0.1729192546583851,130.73046875
+132000,-0.15532943429910362,116.1796875
+133000,-0.1649212541585101,151.35546875
+134000,-0.16318750169519103,119.7734375
+135000,-0.16704425465838513,132.45703125
+136000,-0.1729192546583851,123.5546875
+137000,-0.16904425465838513,134.3984375
+138000,-0.1729192546583851,127.1171875
+139000,-0.1634535677733434,136.2109375
+140000,-0.1689192546583851,130.6171875
+141000,-0.16704425465838513,138.1640625
+142000,-0.1729192546583851,134.3203125
+143000,-0.16904425465838513,136.36328125
+144000,-0.16916925465838506,137.9296875
+145000,-0.1479315638656722,145.6015625
+146000,-0.1709192546583851,121.72265625
+147000,-0.16904425465838513,142.8203125
+148000,-0.1729192546583851,144.4453125
+149000,-0.16904425465838513,144.0390625
+150000,-0.1710440202540846,145.6015625
+151000,-0.1652933014142296,147.09375
+152000,-0.16146079158701876,148.71875
+153000,-0.16716853581853008,147.46875
+154000,-0.170919504627139,149.03515625
+155000,-0.16716853581853008,149.421875
+156000,-0.16505997322125932,151.046875
+157000,-0.16891975459589292,150.796875
+158000,-0.16318750169519103,152.375
+159000,-0.0930342690351822,152.16015625
+160000,-0.16904473897284578,153.8984375
+161000,-0.13979011351574222,154.8359375
+162000,-0.1121616599590477,156.6796875
+163000,-0.1710440202540846,155.86328125
+164000,-0.16705671256034338,157.4921875
+165000,-0.16729284805673472,158.73828125
+166000,-0.16717929957977262,150.0078125
+167000,-0.170919504627139,160.54296875
+168000,-0.15337000798922806,154.10546875
+169000,-0.16343821673423342,160.48828125
+170000,-0.1670559812052913,141.86328125
+171000,-0.16717929957977262,162.43359375
+172000,-0.16904377022283068,162.03125
+173000,-0.16892773663418648,164.12890625
+174000,-0.14535970858032055,165.93359375
+175000,-0.1670559812052913,165.984375
+176000,-0.1729192546583851,145.4765625
+177000,-0.16518375871328217,167.703125
+178000,-0.16704352019157676,147.17578125
+179000,-0.1709192546583851,169.26953125
+180000,-0.1729192546583851,150.46484375
+181000,-0.16904425465838513,170.8125
+182000,-0.09301429029674946,151.43359375
+183000,-0.16505997322125932,172.43359375
+184000,-0.12028109511850016,155.703125
+185000,-0.16131311463592157,174.27734375
+186000,-0.12626838775056426,164.5390625
+187000,-0.16516925465838506,175.953125
+188000,-0.14590236910491416,173.37109375
+189000,-0.1633110689833165,177.50390625
+190000,-0.11265724592476067,185.91796875
+191000,-0.1569252524092285,158.359375
+192000,-0.11839648485177245,185.0703125
+193000,-0.16716853581853008,181.30859375
+194000,-0.13586835645479223,187.8046875
+195000,-0.16704425465838513,162.1328125
+196000,-0.1495909113450118,194.04296875
+197000,-0.1653075750976164,183.203125
+198000,-0.1670559812052913,173.33984375
+199000,-0.1709192546583851,187.38671875
+200000,-0.1671800157313732,186.609375
+201000,-0.16904377022283068,187.78515625
+202000,-0.1710440202540846,179.3671875
+203000,-0.1709192546583851,168.5390625
+204000,-0.16692000456464684,181.28515625
+205000,-0.1729192546583851,170.1796875
+206000,-0.16904425465838513,170.34765625
+207000,-0.16131745825119947,171.87890625
+208000,-0.1630467227913518,193.21875
+209000,-0.17104425465838513,194.45703125
+210000,-0.1709192546583851,194.8515625
+211000,-0.16716925465838506,196.10546875
+212000,-0.170919504627139,182.65234375
+213000,-0.16504425465838513,197.55078125
+214000,-0.1729192546583851,198.44921875
+215000,-0.16704425465838513,199.19140625
+216000,-0.1709192546583851,199.4765625
+217000,-0.1729192546583851,180.05078125
+218000,-0.1590477225414143,212.2734375
+219000,-0.170919504627139,202.42578125
+220000,-0.1572942546583851,208.34765625
+221000,-0.1729192546583851,199.11328125
+222000,-0.16891975459589292,203.98828125
+223000,-0.16905150668583357,203.78515625
+224000,-0.15916925465838505,206.18359375
+225000,-0.16904377022283068,208.1796875
+226000,-0.16904377022283068,207.92578125
+227000,-0.16092525165988436,188.83203125
+228000,-0.15358292731307566,209.49609375
+229000,-0.1650756291603771,190.41015625
+230000,-0.1650452389103536,211.09765625
+231000,-0.1573254422831356,192.0859375
+232000,-0.1689192546583851,192.09765625
+233000,-0.16519916998648865,193.6953125
+234000,-0.16304425465838512,234.7421875
+235000,-0.1573584430064493,216.27734375
+236000,-0.1591709731935682,233.40234375
+237000,-0.170919504627139,197
+238000,-0.16904425465838513,217.6640625
+239000,-0.1670559812052913,198.6328125
+240000,-0.16904377022283068,219.5546875
+241000,-0.17104425465838513,221.15234375
+242000,-0.1729192546583851,200.3203125
+243000,-0.16541737918951782,222.83203125
+244000,-0.15716925465838505,233.5625
+245000,-0.16692000456464684,224.74609375
+246000,-0.16306396523722744,245.30078125
+247000,-0.16891975459589292,219.53515625
+248000,-0.1552964575080289,239.7890625
+249000,-0.16904425465838513,223.8203125
+250000,-0.16704352019157676,227.4375
+251000,-0.16904377022283068,228.26953125
+252000,-0.16320689007116063,209.01171875
+253000,-0.1671800157313732,232.58203125
+254000,-0.16504622291632065,232.5703125
+255000,-0.1709192546583851,233.3359375
+256000,-0.16122694696607742,232.9921875
+257000,Timeout error 

Milestone1/results/content_random_size.csv

 items,uncompressed,compressed
+100,0.06,0.04
+1000,0.59,0.41
+5000,2.98,2.07
+10000,5.95,4.14
+50000,29.78,20.66
+100000,59.57,41.3

Milestone1/results/import_index_uncompressed.csv

 items,time
-0.036
+100,0.029
+1000,0.072
+5000,0.193
+10000,0.33
+50000,1.228
+100000,3.604

Milestone1/results/rss.csv

-items,time 
-2.892
-38.229
+items,time
+100,2.861
+1000,28.462
+5000,183.947
+10000,364.987
+50000,2023.26
+100000,4396.794

Milestone1/results/sitemap.csv

 items,time
-6.274
+100,12.671
+1000,136.022
+5000,832.138
+10000,1865.542
+50000,11905.97

Milestone1/unit_tests/add_csv.js

@@ -22,7 +22,8 @@ var saveAs = args.saveAs,
 browser,
 page,
 ramBefore,
-memBefore;
+memBefore,
+time;
 
 function init_server (){
   var port = 9030;
@@ -45,8 +46,19 @@ function init_server (){
   });
 }
 
+function timeout (){ //Every 30 minutes, check if more than 2 hours have passed since last response
+  setTimeout(function () {
+    if (Date.now() - time > 7200000){ //2 hours
+      process.send("Timeout ERROR");
+    } else {
+      timeout();
+    }
+  }, 1800000); //30 minutes
+}
+
 (async () => {
   init_server();
+  timeout();
   browser = await puppeteer.launch();
   page = await browser.newPage();
   os_utils.cpuUsage(function(v){
@@ -55,6 +67,7 @@ function init_server (){
   diskspace.check('C', function (err, result){
     memBefore = result.used/1024/1024;
   });
+  time = Date.now();
   await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/content_load.html?saveAs='+saveAs);
   const [fileChooser] = await Promise.all([
     page.waitForFileChooser(),

Milestone1/unit_tests/add_random_csv.js

@@ -4,8 +4,6 @@ var os_utils 	= require('os-utils');
 const fs = require('fs');
 const puppeteer = require('puppeteer');
 const Server = require('ws').Server;
-var stream = fs.createWriteStream("./results/test.csv", {flags:'a'});
-fs.truncateSync("./results/test.csv");
 const args = require("yargs")
 .usage("Usage : load_content_random.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
 .demandOption(['saveAs'])
@@ -22,6 +20,7 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
   throw 'Unrecognized save as argument';
 }
 var saveAs = args.saveAs,
+stream,
 browser,
 page,
 result,
@@ -54,7 +53,8 @@ function timeout (){ //Every 30 minutes, check if more than 2 hours have passed
   setTimeout(function () {
     if (Date.now() - time > 7200000){ //2 hours
       stream.write(n + ",Timeout error \n");
-      process.exit();
+      result = [];
+      query();
     } else {
       timeout();
     }
@@ -70,6 +70,7 @@ async function query (){
     return;
   } else {
     n += 1000;
+    console.log(n);
     time = Date.now();
     const [fileChooser] = await Promise.all([
       page.waitForFileChooser(),
@@ -80,6 +81,13 @@ async function query (){
 }
 
 (async () => {
+  if (saveAs === "attachment"){
+    stream = fs.createWriteStream("../results/add_random_csv_attachment.csv", {flags:'a'});
+    fs.truncateSync("../results/add_random_csv_attachment.csv");
+  } else {
+    stream = fs.createWriteStream("../results/add_random_csv_metadata.csv", {flags:'a'});
+    fs.truncateSync("../results/add_random_csv_metadata.csv");
+  }
   stream.write("items,ramUsed,memUsed\n");
   init_server();
   result = gen_random_content(args.p);