Commit 37816bdf authored by Alexandra's avatar Alexandra

new test : add content to multiple indices

parent 8554aa71
......@@ -133,5 +133,15 @@
"titles" : "items,time",
"files_dir" : "./files/SITEMAP/",
"to_pass" : []
},
{
"test_name" : "add content to multiple indices",
"script" : "add_multiple_indices.js",
"link" : "https://softinst115787.host.vifib.net/public/unit_tests/add_multiple_indices.html",
"result_file" : "./results/multiple_indices.csv",
"titles" : "items,timestamp",
"files_dir" : "./files/CSV/",
"paragraphs" : 1000000,
"to_pass" : ["-r", "-n", "10"]
}
]
\ No newline at end of file
]
......@@ -254,4 +254,4 @@ items,timestamp
253000,1582830142425
254000,1582830142536
255000,1582830142639
256000,1582830142748
\ No newline at end of file
256000,1582830142748
......@@ -288,4 +288,4 @@ time,ram,disk
1582830143542,788,31457280
1582830143643,788,31457280
1582830143743,788,31457280
1582830143846,788,31457280
\ No newline at end of file
1582830143846,788,31457280
items,timestamp
1000,1585752977335
2000,1585752977845
3000,1585752978413
4000,1585752978879
5000,1585752979358
6000,1585752979813
7000,1585752980286
8000,1585752980760
9000,1585752981218
10000,1585752981740
11000,1585752982252
12000,1585752982727
13000,1585752983198
14000,1585752983707
15000,1585752984167
16000,1585752984638
17000,1585752985109
18000,1585752985580
19000,1585752986042
20000,1585752986552
21000,1585752987027
22000,1585752987743
23000,1585752988234
24000,1585752988719
25000,1585752989205
26000,1585752989709
27000,1585752990190
28000,1585752990671
29000,1585752991128
30000,1585752991597
31000,1585752992075
32000,1585752992568
33000,1585752993032
34000,1585752993501
35000,1585752993966
36000,1585752994431
37000,1585752994903
38000,1585752995358
39000,1585752995864
40000,1585752996332
41000,1585752996796
42000,1585752997249
43000,1585752997716
44000,1585752998239
45000,1585752998763
46000,1585752999254
47000,1585752999732
48000,1585753000201
49000,1585753000674
50000,1585753001224
51000,1585753001705
52000,1585753002176
53000,1585753002656
54000,1585753003145
55000,1585753003610
56000,1585753004086
57000,1585753004565
58000,1585753005591
59000,1585753006082
60000,1585753006546
61000,1585753007023
62000,1585753007493
63000,1585753007986
64000,1585753008458
65000,1585753008925
66000,1585753009386
67000,1585753009859
68000,1585753010332
69000,1585753010821
70000,1585753011281
71000,1585753011747
72000,1585753012213
73000,1585753012673
74000,1585753013152
75000,1585753013642
76000,1585753014126
77000,1585753014601
78000,1585753015077
79000,1585753015541
80000,1585753016026
81000,1585753016509
82000,1585753017008
83000,1585753017479
84000,1585753017957
85000,1585753018415
86000,1585753018899
87000,1585753019366
88000,1585753019953
89000,1585753020424
90000,1585753021677
91000,1585753022165
92000,1585753022630
93000,1585753023133
94000,1585753023614
95000,1585753024114
96000,1585753024582
97000,1585753025065
98000,1585753025535
99000,1585753026019
100000,1585753026497
101000,1585753026969
102000,1585753027445
103000,1585753027929
104000,1585753028400
105000,1585753028863
106000,1585753029368
107000,1585753029837
108000,1585753030310
109000,1585753030782
110000,1585753031258
111000,1585753031744
112000,1585753032244
113000,1585753032712
114000,1585753033180
115000,1585753033651
116000,1585753034121
117000,1585753034593
118000,1585753035053
119000,1585753035576
120000,1585753036061
121000,1585753037629
122000,1585753038353
123000,1585753038824
124000,1585753039289
125000,1585753039757
126000,1585753040233
127000,1585753040701
128000,1585753041183
129000,1585753041695
130000,1585753042181
131000,1585753042664
132000,1585753043162
133000,1585753044877
134000,1585753045359
135000,1585753045824
136000,1585753046302
137000,1585753046782
138000,1585753047252
139000,1585753047744
140000,1585753048231
141000,1585753048703
142000,1585753049173
143000,1585753049649
144000,1585753050153
145000,1585753050651
146000,1585753051127
147000,1585753051614
148000,1585753052085
149000,1585753052586
150000,1585753053078
151000,1585753053591
152000,1585753054056
153000,1585753054975
154000,1585753056380
155000,1585753056873
156000,1585753057348
157000,1585753057823
158000,1585753058294
159000,1585753058766
160000,1585753059249
161000,1585753059757
162000,1585753060236
163000,1585753060716
164000,1585753061184
165000,1585753061667
166000,1585753062145
167000,1585753062661
168000,1585753063140
169000,1585753063616
170000,1585753064126
171000,1585753064601
172000,1585753065084
173000,1585753067081
174000,1585753067566
175000,1585753068265
176000,1585753068764
177000,1585753069246
178000,1585753069736
179000,1585753070225
180000,1585753070738
181000,1585753073384
182000,1585753074156
183000,1585753074843
184000,1585753075476
185000,1585753076093
186000,1585753076712
187000,1585753077343
188000,1585753078004
189000,1585753078621
190000,1585753079242
191000,1585753080630
192000,1585753082536
193000,1585753083170
194000,1585753083818
195000,1585753084481
196000,1585753085131
197000,1585753085765
198000,1585753086406
199000,1585753087067
200000,1585753089862
201000,1585753090578
202000,1585753091229
203000,1585753091865
204000,1585753092526
205000,1585753093207
206000,1585753093845
207000,1585753094499
208000,1585753095166
209000,1585753097819
210000,1585753098507
211000,1585753099161
212000,1585753099804
213000,1585753100456
214000,1585753101078
215000,1585753101716
216000,1585753102370
217000,1585753102995
218000,1585753103622
219000,1585753104283
220000,1585753108578
221000,1585753109271
222000,1585753109968
223000,1585753110665
224000,1585753111304
225000,1585753111979
226000,1585753112628
227000,1585753115119
228000,1585753116162
229000,1585753116829
230000,1585753117519
231000,1585753118186
232000,1585753118853
233000,1585753121706
234000,1585753122237
235000,1585753122782
236000,1585753123286
237000,1585753123812
238000,1585753124304
239000,1585753127100
240000,1585753127661
241000,1585753128176
242000,1585753128681
243000,1585753129169
244000,1585753131896
245000,1585753132486
246000,1585753133018
247000,1585753133499
248000,1585753133980
249000,1585753136800
250000,1585753137377
251000,1585753137917
252000,1585753140956
253000,1585753141504
254000,1585753142060
255000,1585753142535
256000,1585753143740
This diff is collapsed.
"use strict";
const { exec } = require("child_process");
var childProcess = require("child_process");
var cut_file = require("./random_gen").cut_file;
const puppeteer = require("puppeteer");
const Server = require("ws").Server;
const args = require("yargs")
.usage("Usage : load_content.js -file file_path -n amount")
.demandOption(["file"])
.demandOption(["n"])
.alias("f", "file")
.alias("l", "link")
.describe("file", "file containing content, data must follow the CSV standard")
.describe("link", "link to website treating the data, defaults to Mynij")
.describe("n", "number of parallel indices to add the content to")
.nargs("n", 1)
.nargs("file", 1)
.nargs("link", 1)
.argv;
var link = args.link
? args.link
: "https://softinst115787.host.vifib.net/public/unit_tests/" +
"add_multiple_indices.html";
var files;
var browser;
var page;
var n = 0;
var time;
var monitor;
var args_to_pass = [];
args_to_pass.push("-n");
function init_server (){
var port = 9030;
var ws = new Server({port: port});
ws.on("connection", function(w){
w.on("message", function(){
process.send(n + "," + Date.now());
query();
});
w.on("close", function() {
ws.close();
});
});
}
function timeout (){
//Every 30 minutes, check if more than 2 hours have passed since last response
setTimeout(function () {
if (Date.now() - time > 7200000){ //2 hours
process.send(n + ",Timeout error \n");
files = [];
query();
} else {
timeout();
}
}, 1800000); //30 minutes
}
async function query (){
if (files.length === 0){
monitor.kill();
var ipc=require("node-ipc");
ipc.config.silent = true;
ipc.connectTo("main");
ipc.of.main.emit("message", "DONE");
return;
} else {
n += 1000;
time = Date.now();
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click("input#load")
]);
await fileChooser.accept([files.shift()]);
}
}
(async () => {
files = cut_file(args.file, 1000);
init_server();
browser = await puppeteer.launch();
page = await browser.newPage();
await page.goto(link);
await page.focus("#amount");
await page.keyboard.type(args.n.toString());
await page.$eval('#done', el => el.click());
exec("ps x | grep \"chrome\"", (error, stdout, stderr) => {
args_to_pass.push("-p");
var pids = "";
var processes = stdout.split("\n");
processes.forEach((item) => {
if (!item.includes("grep")) pids += item.trim().split(" ")[0] + " "
});
args_to_pass.push(pids.trim());
monitor = childProcess.fork("./unit_tests/monitor.js", args_to_pass);
timeout();
query();
});
})();
......@@ -7,6 +7,9 @@ var pids = args.p;
var path = args.r
? "./results/add_random_csv_monitor.csv"
: "./results/add_csv_monitor.csv";
if (args.n) {
path = "./results/multiple_indices_monitor.csv";
}
var stream = fs.createWriteStream(path, {flags: "w+"});
stream.write("time,ram,disk\n");
......@@ -43,4 +46,4 @@ process.on("exit", function(){
diskspace.check("/", function (err, result){
start_disk = result.used;
init_monitor();
});
\ No newline at end of file
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment