Commit 36737b80 authored by Alexandra Rogova's avatar Alexandra Rogova

added some bench scripts

parent f3e22b8a
......@@ -32,9 +32,10 @@ function init_server (){
console.log(msg);
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
console.log("Ram used : " + ramUsed + " MB");
//console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore));
//console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
});
browser.close();
});
......
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
function writeResult(items, info){
stream.write(items + "," + info + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/add_csv_attachments.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100k.csv", "-s", "attachment"], amount : 100000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/50k.csv", "-s", "attachment"], amount : 50000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/10k.csv", "-s", "attachment"], amount : 10000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/5000.csv", "-s", "attachment"], amount : 5000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/1000.csv", "-s", "attachment"], amount : 1000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100.csv", "-s", "attachment"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/add_csv_metadata.csv", {flags:'a'});
function writeResult(items, info){
stream.write(items + "," + info + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/add_csv_metadata.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100k.csv", "-s", "metadata"], amount : 100000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/50k.csv", "-s", "metadata"], amount : 50000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/10k.csv", "-s", "metadata"], amount : 10000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/5000.csv", "-s", "metadata"], amount : 5000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/1000.csv", "-s", "metadata"], amount : 1000},
{scriptPath : './add_csv.js', args : ['-f' ,"./test_files/CSV/100.csv", "-s", "metadata"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
......@@ -57,9 +57,10 @@ function init_server (){
console.log(msg);
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
console.log("Ram used : " + ramUsed + " MB");
//console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore));
//console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
});
browser.close();
});
......
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/add_random_csv_attachment.csv", {flags:'a'});
function writeResult(items, info){
stream.write(items + "," + info + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/add_random_csv_attachment.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : './add_random_csv.js', args : ["-p", 100000, "-s", "attachment"], amount : 100000},
{scriptPath : './add_random_csv.js', args : ["-p", 50000, "-s", "attachment"], amount : 50000},
{scriptPath : './add_random_csv.js', args : ["-p", 10000, "-s", "attachment"], amount : 10000},
{scriptPath : './add_random_csv.js', args : ["-p", 5000, "-s", "attachment"], amount : 5000},
{scriptPath : './add_random_csv.js', args : ["-p", 1000, "-s", "attachment"], amount : 1000},
{scriptPath : './add_random_csv.js', args : ["-p", 100, "-s", "attachment"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/add_random_csv_metadata.csv", {flags:'a'});
function writeResult(items, info){
stream.write(items + "," + info + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/add_random_csv_metadata.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : './add_random_csv.js', args : ["-p", 100000, "-s", "metadata"], amount : 100000},
{scriptPath : './add_random_csv.js', args : ["-p", 50000, "-s", "metadata"], amount : 50000},
{scriptPath : './add_random_csv.js', args : ["-p", 10000, "-s", "metadata"], amount : 10000},
{scriptPath : './add_random_csv.js', args : ["-p", 5000, "-s", "metadata"], amount : 5000},
{scriptPath : './add_random_csv.js', args : ["-p", 1000, "-s", "metadata"], amount : 1000},
{scriptPath : './add_random_csv.js', args : ["-p", 100, "-s", "metadata"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
const puppeteer = require('puppeteer');
const getPort = require('get-port');
const Server = require('ws').Server;
const args = require("yargs")
.usage("Usage : add_rss.js -file file_path")
......@@ -13,11 +12,10 @@ var browser,
index_start,
index_total,
db_start,
db_total,
port;
db_total;
function init_server (){
var ws = new Server({port: port});
var ws = new Server({port: 9030});
ws.on('connection', function(w){
w.on('message', function(msg){
if (msg === "Index"){
......@@ -46,11 +44,10 @@ function init_server (){
}
(async () => {
port = await getPort();
init_server();
browser = await puppeteer.launch();
var page = await browser.newPage();
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/add_rss.html?port='+port);
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/add_rss.html');
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
......
......@@ -19,7 +19,6 @@ function init_server (){
if (msg.includes("Done")){
var now = Date.now();
process.send((now-start)/1000);
console.log(msg + " in " + (now-start)/1000 +"s");
ws.close();
browser.close();
} else {
......
......@@ -5,10 +5,13 @@ const args = require("yargs")
.demandOption(['file'])
.alias("f", "file")
.alias("u", "uncompressed")
.aliad("b", "bulk")
.describe("file", "file containing the index")
.describe("uncompressed", "to add if file is uncompressed ! For testing only !")
.describe("bulk", "adds items to db in bulk. Much faster but still in testing")
.nargs("file", 1)
.nargs("u", 0)
.nargs("b", 0)
.argv;
var browser,
......@@ -21,7 +24,8 @@ function init_server (){
w.on('message', function(msg){
if (msg.includes("Done")){
var time_spent = Date.now() - start;
console.log(msg + " in " + time_spent/1000 + "s");
process.send(time_spent/1000);
// console.log(msg + " in " + time_spent/1000 + "s");
browser.close();
} else {
console.log("Error : " + msg);
......@@ -39,6 +43,7 @@ function init_server (){
browser = await puppeteer.launch();
var page = await browser.newPage();
if (args.uncompressed) await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content_uncompressed.html');
else if (args.bulk) await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content_bulk.html');
else await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/import_content.html');
start = Date.now();
const [fileChooser] = await Promise.all([
......
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/import_content.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 100000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 50000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 10000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 5000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 1000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content_bulk.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/import_content_bulk.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 100000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 50000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 10000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 5000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 1000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-b'], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content_uncompressed.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/import_content_uncompressed.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 100000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 50000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 10000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 5000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 1000},
{scriptPath : './import_content.js', args : ['-f' ,"./test_files/CONTENT/.xml", '-u'], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
......@@ -21,7 +21,8 @@ function init_server (){
w.on('message', function(msg){
if (msg.includes("Done : ")){
var time_spent = Date.now() - start;
console.log(msg + " in " + time_spent/1000 + "s");
//console.log(msg + " in " + time_spent/1000 + "s");
process.send(time_spent/1000);
browser.close();
} else {
console.log("Error : " + msg);
......
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_index.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/import_index.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 100000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 50000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 10000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 5000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 1000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml"], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_index_uncompressed.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/import_index_uncompressed.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 100000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 50000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 10000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 5000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 1000},
{scriptPath : './import_index.js', args : ['-f' ,"./test_files/INDEX/.xml", '-u'], amount : 100}];
runScriptSync(to_run, function (err) {if (err) throw err;});
const os = require('os');
const puppeteer = require('puppeteer');
const si = require("systeminformation");
const Server = require('ws').Server;
const args = require("yargs")
.usage("Usage : load_index.js -saveAs metadata/attachment -file file_path")
......@@ -19,6 +21,8 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
var saveAs = args.saveAs,
browser,
page,
ramBefore,
memBefore
adding_start,
adding_total,
saving_start,
......@@ -36,10 +40,16 @@ function init_server (){
saving_start = Date.now();
} else if (msg === "finished saving"){
saving_total = Date.now() - saving_start;
console.log("Time spent :");
console.log("Adding : " + adding_total/1000 + "s");
console.log("Saving : " + saving_total/1000 + "s");
browser.close();
} else if (msg.includes("done !")) {
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
//console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
//console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore) +
"," + adding_total + "," + saving_total);
});
browser.close();
} else {
console.log("Error : " + msg);
browser.close();
......@@ -55,7 +65,11 @@ function init_server (){
init_server();
browser = await puppeteer.launch();
page = await browser.newPage();
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/time_index.html?saveAs='+saveAs);
ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
si.mem(function(data){
memBefore = data.used / 1024 / 1024;
});
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
......
const os = require('os');
const randomstring = require("randomstring");
const tmp = require('tmp');
const fs = require('fs');
const puppeteer = require('puppeteer');
const si = require("systeminformation");
const Server = require('ws').Server;
const args = require("yargs")
.usage("Usage : load_index.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
......@@ -21,6 +23,8 @@ if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
var saveAs = args.saveAs,
browser,
page,
ramBefore,
memBefore
adding_start,
adding_total,
saving_start,
......@@ -61,10 +65,16 @@ function init_server (){
saving_start = Date.now();
} else if (msg === "finished saving"){
saving_total = Date.now() - saving_start;
console.log("Time spent :");
console.log("Adding : " + adding_total/1000 + "s");
console.log("Saving : " + saving_total/1000 + "s");
browser.close();
} else if (msg.includes("done !")) {
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
//console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
//console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
process.send(ramUsed + "," + -(data.used / 1024 / 1024 - memBefore) +
"," + adding_total + "," + saving_total);
});
browser.close();
} else {
console.log("Error : " + msg);
browser.close();
......@@ -91,7 +101,11 @@ function gen_random_content (){
var file_path = gen_random_content();
browser = await puppeteer.launch();
page = await browser.newPage();
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/time_index.html?saveAs='+saveAs);
ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
si.mem(function(data){
memBefore = data.used / 1024 / 1024;
});
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
......
const os = require('os');
const puppeteer = require('puppeteer');
const si = require("systeminformation");
const Server = require('ws').Server;
const args = require("yargs")
.usage("Usage : load_index.js -saveAs metadata/attachment -file file_path")
.demandOption(['file'])
.demandOption(['saveAs'])
.alias("f", "file")
.alias("s", "saveAs")
.describe("file", "file containing content to put in the index, data must follow the CSV standard")
.describe("saveAs", "choose whether to save the index as metadata or attachments in the indexeddb")
.nargs("file", 1)
.nargs("saveAs", 1)
.argv;
if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
throw 'Unrecognized save as argument';
}
var saveAs = args.saveAs,
browser,
page,
ramBefore,
memBefore;
function init_server (){
var port = 9030;
var ws = new Server({port: port});
ws.on('connection', function(w){
w.on('message', function(msg){
console.log(msg);
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
});
browser.close();
});
w.on('close', function() {
ws.close();
});
});
}
(async () => {
init_server();
browser = await puppeteer.launch();
page = await browser.newPage();
ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
si.mem(function(data){
memBefore = data.used / 1024 / 1024;
});
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
]);
await fileChooser.accept([args.file]);
})();
const os = require('os');
const randomstring = require("randomstring");
const tmp = require('tmp');
const fs = require('fs');
const puppeteer = require('puppeteer');
const si = require("systeminformation");
const Server = require('ws').Server;
const args = require("yargs")
.usage("Usage : load_index.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
.demandOption(['saveAs'])
.demandOption(['paragraphs'])
.alias("p", "paragraphs")
.alias("s", "saveAs")
.describe("paragraphs", "Number of random paragraphs to generate")
.describe("saveAs", "choose whether to save the index as metadata or attachments in the indexeddb")
.nargs("saveAs", 1)
.nargs("paragraphs", 1)
.argv;
if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
throw 'Unrecognized save as argument';
}
var saveAs = args.saveAs,
browser,
page,
ramBefore,
memBefore;
function gen_rdm_string (length){
return randomstring.generate({
length: length,
charset: 'alphabetic'
});
}
function gen_rdm_sentence (word_count, word_length){
var tmp_word_array = [];
for (var i = 0; i<word_count; i+=1){
tmp_word_array[i] = gen_rdm_string(word_length);
}
return tmp_word_array.join(" ");
}
function gen_rdm_paragraph (sentence_count, words_per_sentence, word_length){
var tmp_sentence_array = [];
for (var i=0; i<sentence_count; i+=1){
tmp_sentence_array[i] = gen_rdm_sentence(words_per_sentence, word_length);
}
return tmp_sentence_array.join(". ");
}
function init_server (){
var port = 9030;
var ws = new Server({port: port});
ws.on('connection', function(w){
w.on('message', function(msg){
console.log(msg);
var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
var ramUsed = ramAfter - ramBefore;
console.log("Ram used : " + ramUsed + " MB");
si.mem(function(data){
console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
});
browser.close();
});
w.on('close', function() {
ws.close();
});
});
}
function gen_random_content (){
var tmp_para_array = [];
for(var i=0; i<args.p; i+=1){
tmp_para_array[i] = gen_rdm_paragraph(6, 15, 5);
}
var tmp_file = tmp.fileSync();
fs.writeFileSync(tmp_file.name, tmp_para_array.join("\n"));
return tmp_file.name;
}
(async () => {
init_server();
var file_path = gen_random_content();
browser = await puppeteer.launch();
page = await browser.newPage();
ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
si.mem(function(data){
memBefore = data.used / 1024 / 1024;
});
await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
]);
await fileChooser.accept([file_path]);
})();
items,time
items,time
100,3.087
1000,29.468
5000,164.876
......
items,time
135,8.412
......@@ -48,6 +48,7 @@ function runScriptSync(list, callback) {
});
}
fs.truncateSync("./results/rss.csv");
stream.write("items,time \n");
var to_run = [{scriptPath : './add_rss.js', args : ['-f' ,"./test_files/RSS/rss_100000.xml"], amount : 100000},
......
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream("./results/sitemap.csv", {flags:'a'});
function writeResult(items, time){
stream.write(items + "," + time + "\n");
}
function runScriptSync(list, callback) {
if (list.length === 0) return;
var invoked = false,
to_run = list.pop();
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
console.log(to_run.amount + " : " + mes)
writeResult(to_run.amount, mes);
runScriptSync(list, callback);
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
fs.truncateSync("./results/sitemap.csv");
stream.write("items,time \n");
// var to_run = [{scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 100000},
// {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 50000},
// {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 10000},
// {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 5000},
// {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 1000},
// {scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/.xml"], amount : 100}];
var to_run = [{scriptPath : './add_sitemap.js', args : ['-f' ,"./test_files/SITEMAP/44svt_135.xml"], amount : 135}];
runScriptSync(to_run, function (err) {if (err) throw err;});
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment