Skip to content

M
Mynij-unit-tests
Commit 2e1d402d authored by Alexandra Rogova's avatar Alexandra Rogova
Browse files
Options

major restructure part 1

parent 9a5a4eb8
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 108196 additions and 1166625 deletions
.jslintrc 0 → 100644
View file @ 2e1d402d
{
"node" : true,
"maxlen" : 80,
"indent" : 2,
"maxerr" : 3,
"terse" : true,
"for" : true
}
\ No newline at end of file
Milestone1/all.js
View file @ 2e1d402d
const fs = require('fs');
var ipc=require('node-ipc');
const args = require("yargs")
.usage("Usage : Milestone1/all.js -only test -file only on this file -verbose")
.alias("f", "file")
.alias("o", "only")
.alias("v", "verbose")
.describe("file", "If you want to run the test on only one file (as opposed to all files in the directory), add this option and specify file location")
.describe("only", "If you only want to run one test, add this option and specify the test file name (eg : rss_bench)")
.describe("verbose", "Print results to console, if not specified results will only be copied to result file")
.nargs("file", 1)
.nargs("only", 1)
.nargs("verbose", 0)
.argv;
var current;
var to_run = [];
var options = [];
var only_script = args.only == null ? "" : args.only;
if(args.file !== undefined){
options.push("f");
options.push(args.file);
}
if (args.v !== undefined){
options.push("v");
}
ipc.config.id = 'main';
ipc.config.silent = true;
ipc.serve(
function(){
ipc.server.on('message', function(data,socket){
console.log(data);
run(to_run, function (err) {if (err) throw err;});
if (args.v) console.log(data);
if (data == "DONE") run(to_run, function (err) {if (err) throw err;});
});
ipc.server.on('socket.disconnected', function(socket, destroyedSocketID) {
ipc.log('client ' + destroyedSocketID + ' has disconnected!');
......@@ -24,13 +47,12 @@ function run (to_run_array, callback){
if (to_run_array.length === 0){
return;
}
var invoked = false,
current = to_run_array.pop();
current = to_run_array.shift();
console.log("Starting" + current);
process = childProcess.fork(current, []);
process.on('message', function (mes){
});
process = childProcess.fork(current, options);
process.on('message', function (mes){});
process.on('error', function (err) {
if (invoked) return;
......@@ -51,7 +73,9 @@ fs.readdir("./bench_files/", function(err, items){
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (!file.includes("run_scripts_sync")) to_run.push("./bench_files/"+file);
if (!file.includes("run_scripts_sync") && file.includes(only_script)) {
to_run.push("./bench_files/"+file);
}
});
run(to_run, function (err) {if (err) throw err;});
});
Milestone1/bench_files/add_csv_attachments_bench.js
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("../results/add_csv_attachments.csv", {flags:'a'});
fs.truncateSync("../results/add_csv_attachments.csv");
var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
fs.truncateSync("./results/add_csv_attachments.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [];
fs.readdir("../files/CSV/", function(err, items){
fs.readdir("./files/CSV/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "attachment"]});
to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "attachment"]});
});
to_run.sort(function(file1, file2){
var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
return nb_items_file1 - nb_items_file2;
});
run(to_run, "../results/add_csv_attachments.csv", function (err) {if (err) throw err;});
run(to_run, "./results/add_csv_attachments.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/bench.js 0 → 100644
View file @ 2e1d402d
/*jslint es6 */
"use strict";
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var gen_random_content = require("../unit_tests/random_gen").gen_random_content;
var args = JSON.parse(process.argv[2]);
var stream = fs.createWriteStream(args.result_file, {flags: 'w+'});
stream.write(args.titles + "\n");
var to_run = [];
if (args.files_dir) {
fs.readdir(args.files_dir, function (err, items) {
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (args.file === undefined || file.includes(args.file)) {
var to_pass = [...args.to_pass];
to_pass.push("-f");
to_pass.push(args.files_dir + file);
to_run.push({scriptPath: './unit_tests/' + args.script,
args: to_pass});
}
});
console.log("Starting " + args.script);
run(to_run, args.result_file, function (err) {
if (err) {
throw err;
}
});
});
} else {
var random_file = gen_random_content(args.paragraphs);
var to_pass = [...args.to_pass];
to_pass.push("-f");
to_pass.push(random_file);
to_run.push({scriptPath: './unit_tests/' + args.script,
args: to_pass});
run(to_run, args.result_file, function (err) {
if (err) {
throw err;
}
});
}
\ No newline at end of file
Milestone1/bench_files/rss_bench.js
View file @ 2e1d402d
......@@ -10,7 +10,9 @@ fs.readdir("./files/RSS/", function(err, items){
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : './unit_tests/add_rss.js', args : ['-f' ,"./files/RSS/"+file], amount : parseInt(file.substring(21, file.length-4))});
var args = process.argv.includes("v") ? ['-f' ,"./files/RSS/"+file, '-v'] : ['-f' ,"./files/RSS/"+file];
if (! process.argv.includes("f")) to_run.push({scriptPath : './unit_tests/add_rss.js', args : args});
else if (file.includes(process.argv[process.argv.indexOf("f")+1])) to_run.push({scriptPath : './unit_tests/add_rss.js', args : args});
});
run(to_run, "./results/rss.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/run_scripts_sync.js
View file @ 2e1d402d
function get_size(file){
return file.slice(file.indexOf("_")+1, file.lastIndexOf("."));
/*jslint es6 */
"use strict";
function get_size(file) {
return file.slice(file.indexOf("_") + 1, file.lastIndexOf("."));
}
module.exports = {
run: function (args, filepath, callback) {
var ipc=require('node-ipc');
ipc.config.silent = true;
ipc.connectTo("main");
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream(filepath, {flags:'a'});
function write (message){
if (filepath === "") return;
stream.write(message + "\n");
}
run: function (args, filepath, callback) {
var ipc = require('node-ipc');
ipc.config.silent = true;
ipc.connectTo("main");
if (args.length === 0){
ipc.of.main.emit('message', "DONE");
return;
}
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream(filepath, {flags: 'a'});
function write(message) {
if (filepath === "") {
return;
}
stream.write(message + "\n");
}
if (args.length === 0) {
ipc.of.main.emit('message', "DONE");
return;
}
var invoked = false,
to_run = args.shift();
console.log(to_run.args[1]);
process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes){
mes = mes.toString();
if (mes.includes("DONE")){
module.exports.run(args, filepath, callback);
} else if (mes.includes("ERROR")){
if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
else write(mes);
module.exports.run(args, filepath, callback);
} else {
if (to_run.args.includes("-f")) write(get_size(to_run.args[1]) + "," + mes);
else write(mes);
}
});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
}
var invoked = false;
var to_run = args.shift();
console.log(to_run.args);
var process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes) {
mes = mes.toString();
if (mes.includes("DONE")) {
module.exports.run(args, filepath, callback);
} else if (mes.includes("ERROR")) {
ipc.of.main.emit('message', mes);
if (to_run.args.includes("-f")) {
write(get_size(to_run.args[1]) + "," + mes);
} else {
write(mes);
}
module.exports.run(args, filepath, callback);
} else {
ipc.of.main.emit('message', mes);
if (to_run.args.includes("-f")) {
write(get_size(to_run.args[1]) + "," + mes);
} else {
write(mes);
}
}
});
process.on('error', function (err) {
if (invoked) {
return;
}
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) {
return;
}
invoked = true;
var err = code === 0
? null
: new Error('exit code ' + code);
callback(err);
});
}
};
Milestone1/config.json 0 → 100644
View file @ 2e1d402d
[
{
"test_name" : "add csv attachments",
"script" : "add_csv.js",
"link" : "https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs=",
"result_file" : "./results/add_csv_attachments.csv",
"titles" : "items,ramUsed,diskUsed",
"files_dir" : "./files/CSV/",
"to_pass" : ["-s", "attachment"]
},
{
"test_name" : "add random csv attachments",
"script" : "add_csv.js",
"link" : "https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs=",
"result_file" : "./results/add_random_csv_attachments.csv",
"titles" : "items,ramUsed,diskUsed",
"paragraphs" : 1000000,
"to_pass" : ["-s", "attachment"]
},
{
"test_name" : "add csv metadata",
"script" : "add_csv.js",
"link" : "https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs=",
"result_file" : "./results/add_csv_metadata.csv",
"titles" : "items,ramUsed,diskUsed",
"files_dir" : "./files/CSV/",
"to_pass" : ["-s", "metadata"]
},
{
"test_name" : "add random csv metadata",
"script" : "add_csv.js",
"link" : "https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs=",
"result_file" : "./results/add_random_csv_metadata.csv",
"titles" : "items,ramUsed,diskUsed",
"paragraphs" : 1000000,
"to_pass" : ["-s", "metadata"]
},
{
"test_name" : "add rss",
"script" : "add_rss.js",
"result_file" : "./results/rss.csv",
"titles" : "items,time",
"files_dir" : "./files/RSS/",
"to_pass" : []
}
]
\ No newline at end of file
Milestone1/csv_1000000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/files/CSV/csv_100.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/files/CSV/csv_1000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This source diff could not be displayed because it is too large. You can view the blob instead.
Milestone1/files/CSV/csv_10000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/files/CSV/csv_100000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/files/CSV/csv_5000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/files/CSV/csv_50000.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/results/add_csv_attachments.csv
View file @ 2e1d402d
items,ramUsed,memUsed
100,-0.06638329791223896,2.56640625
1000,-0.16946998114826495,4.65234375
5000,-0.30526011830635114,13.390625
10000,-0.10023595024397591,22.12890625
50000,-0.003956315988130288,32.82421875
100000,Timeout ERROR
items,ramUsed,diskUsed
Milestone1/results/add_random_csv_attachment.csv deleted 100644 → 0
View file @ 9a5a4eb8
This diff is collapsed.
Milestone1/results/rss.csv
View file @ 2e1d402d
items,time
100,2.861
1000,28.462
5000,183.947
10000,364.987
50000,2023.26
100000,4396.794
Milestone1/results/sitemap.csv
View file @ 2e1d402d
items,time
100,12.671
1000,136.022
5000,832.138
10000,1865.542
50000,11905.97
100,6.689
Milestone1/test.js 0 → 100644
View file @ 2e1d402d
/*jslint es6 */
"use strict";
const fs = require('fs');
var ipc = require('node-ipc');
const args = require("yargs")
.usage("Usage : Milestone1/all.js -only test -file only on this file -v")
.alias("f", "file")
.alias("o", "only")
.alias("v", "verbose")
.describe("file", "If you want to run the test on only one file" +
"(as opposed to all files in the directory), " +
"add this option and specify file location")
.describe("only", "If you only want to run one test, " +
"add this option and specify the test file name (eg : rss_bench)")
.describe("verbose", "Print results to console, " +
"if not specified results will only be copied to result file")
.nargs("file", 1)
.nargs("only", 1)
.nargs("verbose", 0)
.argv;
var run;
var to_run = [];
var options = [];
var only_script = args.only == null
? ""
: args.only;
if (args.file !== undefined) {
options.file = args.file;
}
ipc.config.id = 'main';
ipc.config.silent = true;
ipc.serve(
function () {
ipc.server.on('message', function (data) {
if (args.v) {
console.log(data);
}
if (data === "DONE") {
run(to_run, function (err) {
if (err) {
throw err;
}
});
}
});
}
);
ipc.server.start();
run = function (callback) {
var childProcess = require('child_process');
if (to_run.length === 0) {
return;
}
var invoked = false;
var process = childProcess.fork("./bench_files/bench.js",
[JSON.stringify(to_run.shift())]);
process.on('error', function (err) {
if (invoked) {
return;
}
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) {
return;
}
invoked = true;
var err = code === 0
? null
: new Error('exit code ' + code);
callback(err);
});
};
fs.readFile("./config.json", 'utf8', function (err, data) {
if (err) {
return console.log('Error while reading config file : ' + err);
}
var test_args = JSON.parse(data);
test_args.forEach(function (test) {
if (test.test_name.includes(only_script)) {
var tmp = test;
tmp.file = options.file;
to_run.push(tmp);
}
});
run(function (err) {
if (err) {
throw err;
}
});
});
Milestone1/unit_tests/add_csv.js
View file @ 2e1d402d
/*jslint es6 */
"use strict";
var cut_file = require("./random_gen").cut_file;
var diskspace = require('diskspace');
var os_utils = require('os-utils');
var os_utils = require('os-utils');
const fs = require('fs');
const puppeteer = require('puppeteer');
const Server = require('ws').Server;
const args = require("yargs")
......@@ -8,70 +13,97 @@ const args = require("yargs")
.demandOption(['saveAs'])
.alias("f", "file")
.alias("s", "saveAs")
.alias("l", "link")
.describe("file", "file containing content, data must follow the CSV standard")
.describe("saveAs", "choose whether to save the content as metadata or attachments in the indexeddb")
.describe("saveAs", "choose whether to save the content as " +
"metadata or attachments in the indexeddb")
.describe("link", "link to website treating the data, defaults to Mynij")
.nargs("file", 1)
.nargs("saveAs", 1)
.nargs("link", 1)
.argv;
if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
throw 'Unrecognized save as argument';
throw 'Unrecognized save as argument';
}
var saveAs = args.saveAs,
browser,
page,
ramBefore,
memBefore,
time;
var link = args.link
? args.link
: "https://softinst115787.host.vifib.net/public/unit_tests/" +
"content_load.html?saveAs=";
var saveAs = args.saveAs;
var files;
var stream;
var browser;
var page;
var n = 0;
var time;
var ramBefore;
var memBefore;
function init_server (){
var port = 9030;
var ws = new Server({port: port});
ws.on('connection', function(w){
w.on('message', function(msg){
diskspace.check('C', function (err, result){
var memUsed = (result.used/1024/1024) - memBefore;
os_utils.cpuUsage(function(v){
var ramUsed = v - ramBefore;
process.send(ramUsed + "," + memUsed);
process.send("DONE");
var port = 9030;
var ws = new Server({port: port});
ws.on('connection', function(w){
w.on('message', function(msg){
console.log(msg);
diskspace.check('C', function (err, result){
var memUsed = (result.used/1024/1024) - memBefore;
os_utils.cpuUsage(function(v){
var ramUsed = v - ramBefore;
process.send(n + "," + ramUsed + "," + memUsed + "\n");
query();
});
});
});
w.on('close', function() {
ws.close();
});
});
browser.close();
});
w.on('close', function() {
ws.close();
});
});
}
function timeout (){ //Every 30 minutes, check if more than 2 hours have passed since last response
setTimeout(function () {
if (Date.now() - time > 7200000){ //2 hours
process.send("Timeout ERROR");
function timeout (){
//Every 30 minutes, check if more than 2 hours have passed since last response
setTimeout(function () {
if (Date.now() - time > 7200000){ //2 hours
process.send(n + ",Timeout error \n");
result = [];
query();
} else {
timeout();
}
}, 1800000); //30 minutes
}
async function query (){
if (files.length === 0){
var ipc=require('node-ipc');
ipc.config.silent = true;
ipc.connectTo("main");
ipc.of.main.emit('message', "DONE");
return;
} else {
timeout();
n += 1000;
time = Date.now();
const [fileChooser] = await Promise.all([
page.waitForFileChooser(),
page.click('input#load')
]);
await fileChooser.accept([files.shift()]);
}
}, 1800000); //30 minutes
}
(async () => {
init_server();
timeout();
browser = await puppeteer.launch();
page = await browser.newPage();
os_utils.cpuUsage(function(v){
ramBefore = v;
});