new test : add content to multiple indices

Milestone1/config.json

@@ -133,5 +133,15 @@
         "titles" : "items,time",
         "files_dir" : "./files/SITEMAP/",
         "to_pass" : []
+    },
+    {
+        "test_name" : "add content to multiple indices",
+        "script" : "add_multiple_indices.js",
+        "link" : "https://softinst115787.host.vifib.net/public/unit_tests/add_multiple_indices.html",
+        "result_file" : "./results/multiple_indices.csv",
+        "titles" : "items,timestamp",
+        "files_dir" : "./files/CSV/",
+        "paragraphs" : 1000000,
+        "to_pass" : ["-r", "-n", "10"]
     }
-]
\ No newline at end of file
+]

Milestone1/results/add_csv.csv

@@ -254,4 +254,4 @@ items,timestamp
 253000,1582830142425
 254000,1582830142536
 255000,1582830142639
-256000,1582830142748
\ No newline at end of file
+256000,1582830142748

Milestone1/results/add_csv_monitor.csv

@@ -288,4 +288,4 @@ time,ram,disk
 1582830143542,788,31457280
 1582830143643,788,31457280
 1582830143743,788,31457280
-1582830143846,788,31457280
\ No newline at end of file
+1582830143846,788,31457280

Milestone1/results/multiple_indices.csv

+items,timestamp
+1000,1585752977335
+2000,1585752977845
+3000,1585752978413
+4000,1585752978879
+5000,1585752979358
+6000,1585752979813
+7000,1585752980286
+8000,1585752980760
+9000,1585752981218
+10000,1585752981740
+11000,1585752982252
+12000,1585752982727
+13000,1585752983198
+14000,1585752983707
+15000,1585752984167
+16000,1585752984638
+17000,1585752985109
+18000,1585752985580
+19000,1585752986042
+20000,1585752986552
+21000,1585752987027
+22000,1585752987743
+23000,1585752988234
+24000,1585752988719
+25000,1585752989205
+26000,1585752989709
+27000,1585752990190
+28000,1585752990671
+29000,1585752991128
+30000,1585752991597
+31000,1585752992075
+32000,1585752992568
+33000,1585752993032
+34000,1585752993501
+35000,1585752993966
+36000,1585752994431
+37000,1585752994903
+38000,1585752995358
+39000,1585752995864
+40000,1585752996332
+41000,1585752996796
+42000,1585752997249
+43000,1585752997716
+44000,1585752998239
+45000,1585752998763
+46000,1585752999254
+47000,1585752999732
+48000,1585753000201
+49000,1585753000674
+50000,1585753001224
+51000,1585753001705
+52000,1585753002176
+53000,1585753002656
+54000,1585753003145
+55000,1585753003610
+56000,1585753004086
+57000,1585753004565
+58000,1585753005591
+59000,1585753006082
+60000,1585753006546
+61000,1585753007023
+62000,1585753007493
+63000,1585753007986
+64000,1585753008458
+65000,1585753008925
+66000,1585753009386
+67000,1585753009859
+68000,1585753010332
+69000,1585753010821
+70000,1585753011281
+71000,1585753011747
+72000,1585753012213
+73000,1585753012673
+74000,1585753013152
+75000,1585753013642
+76000,1585753014126
+77000,1585753014601
+78000,1585753015077
+79000,1585753015541
+80000,1585753016026
+81000,1585753016509
+82000,1585753017008
+83000,1585753017479
+84000,1585753017957
+85000,1585753018415
+86000,1585753018899
+87000,1585753019366
+88000,1585753019953
+89000,1585753020424
+90000,1585753021677
+91000,1585753022165
+92000,1585753022630
+93000,1585753023133
+94000,1585753023614
+95000,1585753024114
+96000,1585753024582
+97000,1585753025065
+98000,1585753025535
+99000,1585753026019
+100000,1585753026497
+101000,1585753026969
+102000,1585753027445
+103000,1585753027929
+104000,1585753028400
+105000,1585753028863
+106000,1585753029368
+107000,1585753029837
+108000,1585753030310
+109000,1585753030782
+110000,1585753031258
+111000,1585753031744
+112000,1585753032244
+113000,1585753032712
+114000,1585753033180
+115000,1585753033651
+116000,1585753034121
+117000,1585753034593
+118000,1585753035053
+119000,1585753035576
+120000,1585753036061
+121000,1585753037629
+122000,1585753038353
+123000,1585753038824
+124000,1585753039289
+125000,1585753039757
+126000,1585753040233
+127000,1585753040701
+128000,1585753041183
+129000,1585753041695
+130000,1585753042181
+131000,1585753042664
+132000,1585753043162
+133000,1585753044877
+134000,1585753045359
+135000,1585753045824
+136000,1585753046302
+137000,1585753046782
+138000,1585753047252
+139000,1585753047744
+140000,1585753048231
+141000,1585753048703
+142000,1585753049173
+143000,1585753049649
+144000,1585753050153
+145000,1585753050651
+146000,1585753051127
+147000,1585753051614
+148000,1585753052085
+149000,1585753052586
+150000,1585753053078
+151000,1585753053591
+152000,1585753054056
+153000,1585753054975
+154000,1585753056380
+155000,1585753056873
+156000,1585753057348
+157000,1585753057823
+158000,1585753058294
+159000,1585753058766
+160000,1585753059249
+161000,1585753059757
+162000,1585753060236
+163000,1585753060716
+164000,1585753061184
+165000,1585753061667
+166000,1585753062145
+167000,1585753062661
+168000,1585753063140
+169000,1585753063616
+170000,1585753064126
+171000,1585753064601
+172000,1585753065084
+173000,1585753067081
+174000,1585753067566
+175000,1585753068265
+176000,1585753068764
+177000,1585753069246
+178000,1585753069736
+179000,1585753070225
+180000,1585753070738
+181000,1585753073384
+182000,1585753074156
+183000,1585753074843
+184000,1585753075476
+185000,1585753076093
+186000,1585753076712
+187000,1585753077343
+188000,1585753078004
+189000,1585753078621
+190000,1585753079242
+191000,1585753080630
+192000,1585753082536
+193000,1585753083170
+194000,1585753083818
+195000,1585753084481
+196000,1585753085131
+197000,1585753085765
+198000,1585753086406
+199000,1585753087067
+200000,1585753089862
+201000,1585753090578
+202000,1585753091229
+203000,1585753091865
+204000,1585753092526
+205000,1585753093207
+206000,1585753093845
+207000,1585753094499
+208000,1585753095166
+209000,1585753097819
+210000,1585753098507
+211000,1585753099161
+212000,1585753099804
+213000,1585753100456
+214000,1585753101078
+215000,1585753101716
+216000,1585753102370
+217000,1585753102995
+218000,1585753103622
+219000,1585753104283
+220000,1585753108578
+221000,1585753109271
+222000,1585753109968
+223000,1585753110665
+224000,1585753111304
+225000,1585753111979
+226000,1585753112628
+227000,1585753115119
+228000,1585753116162
+229000,1585753116829
+230000,1585753117519
+231000,1585753118186
+232000,1585753118853
+233000,1585753121706
+234000,1585753122237
+235000,1585753122782
+236000,1585753123286
+237000,1585753123812
+238000,1585753124304
+239000,1585753127100
+240000,1585753127661
+241000,1585753128176
+242000,1585753128681
+243000,1585753129169
+244000,1585753131896
+245000,1585753132486
+246000,1585753133018
+247000,1585753133499
+248000,1585753133980
+249000,1585753136800
+250000,1585753137377
+251000,1585753137917
+252000,1585753140956
+253000,1585753141504
+254000,1585753142060
+255000,1585753142535
+256000,1585753143740

Milestone1/unit_tests/add_multiple_indices.js

+"use strict";
+
+const { exec } = require("child_process");
+var childProcess = require("child_process");
+var cut_file = require("./random_gen").cut_file;
+const puppeteer = require("puppeteer");
+const Server = require("ws").Server;
+const args = require("yargs")
+.usage("Usage : load_content.js -file file_path -n amount")
+.demandOption(["file"])
+.demandOption(["n"])
+.alias("f", "file")
+.alias("l", "link")
+.describe("file", "file containing content, data must follow the CSV standard")
+.describe("link", "link to website treating the data, defaults to Mynij")
+.describe("n", "number of parallel indices to add the content to")
+.nargs("n", 1)
+.nargs("file", 1)
+.nargs("link", 1)
+.argv;
+
+var link = args.link
+? args.link
+: "https://softinst115787.host.vifib.net/public/unit_tests/" +
+                            "add_multiple_indices.html";
+
+var files;
+var browser;
+var page;
+var n = 0;
+var time;
+var monitor;
+var args_to_pass = [];
+args_to_pass.push("-n");
+
+function init_server (){
+  var port = 9030;
+  var ws = new Server({port: port});
+  ws.on("connection", function(w){
+    w.on("message", function(){
+      process.send(n + "," + Date.now());
+      query();
+    });
+    w.on("close", function() {
+      ws.close();
+    });
+  });
+}
+
+function timeout (){
+  //Every 30 minutes, check if more than 2 hours have passed since last response
+  setTimeout(function () {
+    if (Date.now() - time > 7200000){ //2 hours
+      process.send(n + ",Timeout error \n");
+      files = [];
+      query();
+    } else {
+      timeout();
+    }
+  }, 1800000); //30 minutes
+}
+
+async function query (){
+  if (files.length === 0){
+    monitor.kill();
+    var ipc=require("node-ipc");
+    ipc.config.silent = true;
+    ipc.connectTo("main");
+    ipc.of.main.emit("message", "DONE");
+    return;
+  } else {
+    n += 1000;
+    time = Date.now();
+    const [fileChooser] = await Promise.all([
+      page.waitForFileChooser(),
+      page.click("input#load")
+    ]);
+    await fileChooser.accept([files.shift()]);
+  }
+}
+
+(async () => {
+
+  files = cut_file(args.file, 1000);
+  init_server();
+  browser = await puppeteer.launch();
+  page = await browser.newPage();
+  await page.goto(link);
+  await page.focus("#amount");
+  await page.keyboard.type(args.n.toString());
+  await page.$eval('#done', el => el.click());
+
+  exec("ps x | grep \"chrome\"", (error, stdout, stderr) => {
+     args_to_pass.push("-p");
+     var pids = "";
+     var processes = stdout.split("\n");
+     processes.forEach((item) => {
+       if (!item.includes("grep")) pids += item.trim().split(" ")[0] + " "
+     });
+
+     args_to_pass.push(pids.trim());
+     monitor = childProcess.fork("./unit_tests/monitor.js", args_to_pass);
+     timeout();
+     query();
+   });
+
+})();

Milestone1/unit_tests/monitor.js

@@ -7,6 +7,9 @@ var pids = args.p;
 var path = args.r
     ? "./results/add_random_csv_monitor.csv"
     : "./results/add_csv_monitor.csv";
+if (args.n) {
+    path = "./results/multiple_indices_monitor.csv";
+}
 var stream = fs.createWriteStream(path, {flags: "w+"});
 stream.write("time,ram,disk\n");
 
@@ -43,4 +46,4 @@ process.on("exit", function(){
 diskspace.check("/", function (err, result){
     start_disk = result.used;
     init_monitor();
-});
\ No newline at end of file
+});