Skip to content

M
Mynij-unit-tests
Commit a6e13c59 authored by Alexandra Rogova's avatar Alexandra Rogova
Browse files
Options

major restructure part 2

parent 2e1d402d
Hide whitespace changes
Inline Side-by-side
Showing with 69481 additions and 1490 deletions

Too many changes to show.
Plain diff Email patch

To preserve performance only 371 of 371+ files are displayed.

.eslintrc.json 0 → 100644
View file @ a6e13c59
{
"env": {
"amd" : true,
"node": true,
"es6": true
},
"extends": "eslint:recommended",
"globals": {
"Atomics": "readonly",
"SharedArrayBuffer": "readonly"
},
"parserOptions": {
"ecmaVersion": 2018,
"sourceType": "module"
},
"rules": {
"indent": [
"error",
4
],
"linebreak-style": [
"error",
"windows"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
]
}
}
\ No newline at end of file
.jslintrc deleted 100644 → 0
View file @ 2e1d402d
{
"node" : true,
"maxlen" : 80,
"indent" : 2,
"maxerr" : 3,
"terse" : true,
"for" : true
}
\ No newline at end of file
Milestone1/all.js deleted 100644 → 0
View file @ 2e1d402d
const fs = require('fs');
var ipc=require('node-ipc');
const args = require("yargs")
.usage("Usage : Milestone1/all.js -only test -file only on this file -verbose")
.alias("f", "file")
.alias("o", "only")
.alias("v", "verbose")
.describe("file", "If you want to run the test on only one file (as opposed to all files in the directory), add this option and specify file location")
.describe("only", "If you only want to run one test, add this option and specify the test file name (eg : rss_bench)")
.describe("verbose", "Print results to console, if not specified results will only be copied to result file")
.nargs("file", 1)
.nargs("only", 1)
.nargs("verbose", 0)
.argv;
var current;
var to_run = [];
var options = [];
var only_script = args.only == null ? "" : args.only;
if(args.file !== undefined){
options.push("f");
options.push(args.file);
}
if (args.v !== undefined){
options.push("v");
}
ipc.config.id = 'main';
ipc.config.silent = true;
ipc.serve(
function(){
ipc.server.on('message', function(data,socket){
if (args.v) console.log(data);
if (data == "DONE") run(to_run, function (err) {if (err) throw err;});
});
ipc.server.on('socket.disconnected', function(socket, destroyedSocketID) {
ipc.log('client ' + destroyedSocketID + ' has disconnected!');
});
}
);
ipc.server.start();
function run (to_run_array, callback){
var childProcess = require('child_process');
if (to_run_array.length === 0){
return;
}
var invoked = false,
current = to_run_array.shift();
console.log("Starting" + current);
process = childProcess.fork(current, options);
process.on('message', function (mes){});
process.on('error', function (err) {
if (invoked) return;
invoked = true;
callback(err);
});
process.on('exit', function (code) {
if (invoked) return;
invoked = true;
var err = code === 0 ? null : new Error('exit code ' + code);
callback(err);
});
};
fs.readdir("./bench_files/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (!file.includes("run_scripts_sync") && file.includes(only_script)) {
to_run.push("./bench_files/"+file);
}
});
run(to_run, function (err) {if (err) throw err;});
});
Milestone1/bench_files/add_csv_attachments_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/add_csv_attachments.csv", {flags:'a'});
fs.truncateSync("./results/add_csv_attachments.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [];
fs.readdir("./files/CSV/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : './unit_tests/add_csv.js', args : ['-f' ,"./files/CSV/"+file, "-s", "attachment"]});
});
to_run.sort(function(file1, file2){
var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
return nb_items_file1 - nb_items_file2;
});
run(to_run, "./results/add_csv_attachments.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/add_csv_metadata_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("../results/add_csv_metadata.csv", {flags:'a'});
fs.truncateSync("../results/add_csv_metadata.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [];
fs.readdir("../files/CSV/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : '../unit_tests/add_csv.js', args : ['-f' ,"../files/CSV/"+file, "-s", "metadata"], amount : parseInt(file.substring(17, file.length-4))});
});
to_run.sort(function(file1, file2){
var nb_items_file1 = parseInt(file1.args[1].slice(file1.args[1].indexOf("_")+1, file1.args[1].lastIndexOf(".")));
var nb_items_file2 = parseInt(file2.args[1].slice(file2.args[1].indexOf("_")+1, file2.args[1].lastIndexOf(".")));
return nb_items_file1 - nb_items_file2;
});
run(to_run, "../results/add_csv_metadata.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/add_random_csv_attachments_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("../results/add_random_csv_attachment.csv", {flags:'a'});
fs.truncateSync("../results/add_random_csv_attachment.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "attachment"], amount : 1000000}];
run(to_run, "../results/add_random_csv_attachment.csv", function (err) {if (err) throw err;});
Milestone1/bench_files/add_random_csv_metadata_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("../results/add_random_csv_metadata.csv", {flags:'a'});
fs.truncateSync("../results/add_random_csv_metadata.csv");
stream.write("items,ramUsed,memUsed\n");
var to_run = [{scriptPath : '../unit_tests/add_random_csv.js', args : ["-p", 1000000, "-s", "metadata"], amount : 1000000}];
run(to_run, "../results/add_random_csv_metadata.csv", function (err) {if (err) throw err;});
Milestone1/bench_files/bench.js
View file @ a6e13c59
/*jslint es6 */
"use strict";
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var run = require("./run_scripts_sync").run;
const fs = require("fs");
var gen_random_content = require("../unit_tests/random_gen").gen_random_content;
var args = JSON.parse(process.argv[2]);
var stream = fs.createWriteStream(args.result_file, {flags: 'w+'});
var stream = fs.createWriteStream(args.result_file, {flags: "w+"});
stream.write(args.titles + "\n");
var to_run = [];
......@@ -13,15 +13,15 @@ var to_run = [];
if (args.files_dir) {
fs.readdir(args.files_dir, function (err, items) {
if (err) {
return console.log('Unable to scan directory: ' + err);
return console.log("Unable to scan directory: " + err);
}
items.forEach(function (file) {
if (args.file === undefined || file.includes(args.file)) {
var to_pass = [...args.to_pass];
to_pass.push("-f");
to_pass.push(args.files_dir + file);
to_run.push({scriptPath: './unit_tests/' + args.script,
args: to_pass});
to_run.push({scriptPath: "./unit_tests/" + args.script,
args: to_pass});
}
});
console.log("Starting " + args.script);
......@@ -36,8 +36,8 @@ if (args.files_dir) {
var to_pass = [...args.to_pass];
to_pass.push("-f");
to_pass.push(random_file);
to_run.push({scriptPath: './unit_tests/' + args.script,
args: to_pass});
to_run.push({scriptPath: "./unit_tests/" + args.script,
args: to_pass});
run(to_run, args.result_file, function (err) {
if (err) {
throw err;
......
Milestone1/bench_files/download_content_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/content_csv_size.csv", {flags:'a'});
fs.truncateSync("./results/content_csv_size.csv");
stream.write("items,uncompressed,compressed\n");
var to_run = [];
fs.readdir("./files/CSV/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : './unit_tests/download_content.js', args : ['-f' ,"./files/CSV/"+file], amount : parseInt(file.substring(17, file.length-4))});
});
run(to_run, "./results/content_csv_size.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/download_content_random_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
var to_run = [{scriptPath : './unit_tests/download_content_random.js', args : ["-p", 1000000], amount : 1000000}];
run(to_run, "./results/content_random_size.csv", function (err) {if (err) throw err;});
Milestone1/bench_files/download_index_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/index_csv_size.csv", {flags:'a'});
fs.truncateSync("./results/index_csv_size.csv");
stream.write("items,uncompressed,compressed\n");
var to_run = [];
fs.readdir("./files/CSV/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
to_run.push({scriptPath : './unit_tests/download_index.js', args : ['-f' ,"./files/CSV/"+file], amount : parseInt(file.substring(17, file.length-4))});
});
run(to_run, "./results/index_csv_size.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/download_index_random_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
var to_run = [{scriptPath : './unit_tests/download_index_random.js', args : ["-p", 1000000], amount : 1000000}];
run(to_run, "./results/index_random_size.csv", function (err) {if (err) throw err;});
Milestone1/bench_files/export_content_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/export_content.csv", {flags:'a'});
fs.truncateSync("./results/export_content.csv");
stream.write("items,zip,json\n");
var to_run = [];
fs.readdir("./files/CONTENT/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes("zip")) to_run.push({scriptPath : './unit_tests/export_content.js', args : ['-f' ,"./files/CONTENT/"+file], amount : parseInt(file.substring(25, file.length-4))});
});
run(to_run, "./results/export_content.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/export_index_bench.js deleted 100644 → 0
View file @ 2e1d402d
const fs = require('fs');
var stream = fs.createWriteStream("./results/export_index.csv", {flags:'a'});
fs.truncateSync("./results/export_index.csv");
stream.write("items,zip,json\n");
var to_run = [];
fs.readdir("./files/INDEX/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes(".json")) to_run.push({scriptPath : './unit_tests/export_index.js', args : ['-f' ,"./files/INDEX/"+file], amount : parseInt(file.substring(21, file.length-5))});
});
run(to_run, "./results/export_index.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/import_content_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content.csv", {flags:'a'});
fs.truncateSync("./results/import_content.csv");
stream.write("items,time\n");
var to_run = [];
fs.readdir("./files/CONTENT/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes("zip")) to_run.push({scriptPath : './unit_tests/import_content.js', args : ['-f' ,"./files/CONTENT/"+file], amount : parseInt(file.substring(25, file.length-4))});
});
run(to_run, "./results/import_content.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/import_content_bulk_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content_bulk.csv", {flags:'a'});
fs.truncateSync("./results/import_content_bulk.csv");
stream.write("items,time\n");
var to_run = [];
fs.readdir("./files/CONTENT/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes(".zip")) to_run.push({scriptPath : './unit_tests/import_content.js', args : ['-f' ,"./files/CONTENT/"+file, '-b'], amount : parseInt(file.substring(17, file.length-4))});
});
run(to_run, "./results/import_content_bulk.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/import_content_uncompressed_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_content_uncompressed.csv", {flags:'a'});
fs.truncateSync("./results/import_content_uncompressed.csv");
stream.write("items,time\n");
var to_run = [];
fs.readdir("./files/CONTENT/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes(".json")) to_run.push({scriptPath : './unit_tests/import_content.js', args : ['-f' ,"./files/CONTENT/"+file, '-u'], amount : parseInt(file.substring(17, file.length-5))});
});
run(to_run, "./results/import_content_uncompressed.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/import_index_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_index.csv", {flags:'a'});
fs.truncateSync("./results/import_index.csv");
stream.write("items,time\n");
var to_run = [];
fs.readdir("./files/INDEX/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes(".zip")) to_run.push({scriptPath : './unit_tests/import_index.js', args : ['-f' ,"./files/INDEX/"+file], amount : parseInt(file.substring(21, file.length-4))});
});
run(to_run, "./results/import_index.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/import_index_uncompressed_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/import_index_uncompressed.csv", {flags:'a'});
fs.truncateSync("./results/import_index_uncompressed.csv");
stream.write("items,time\n");
var to_run = [];
fs.readdir("./files/INDEX/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
if (file.includes(".json")) to_run.push({scriptPath : './unit_tests/import_index.js', args : ['-f' ,"./files/INDEX/"+file, "-u"], amount : parseInt(file.substring(21, file.length-5))});
});
run(to_run, "./results/import_index_uncompressed.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/rss_bench.js deleted 100644 → 0
View file @ 2e1d402d
var run = require('./run_scripts_sync').run;
const fs = require('fs');
var stream = fs.createWriteStream("./results/rss.csv", {flags:'a'});
fs.truncateSync("./results/rss.csv");
stream.write("items,time \n");
var to_run = [];
fs.readdir("./files/RSS/", function(err, items){
if (err) {
return console.log('Unable to scan directory: ' + err);
}
items.forEach(function (file) {
var args = process.argv.includes("v") ? ['-f' ,"./files/RSS/"+file, '-v'] : ['-f' ,"./files/RSS/"+file];
if (! process.argv.includes("f")) to_run.push({scriptPath : './unit_tests/add_rss.js', args : args});
else if (file.includes(process.argv[process.argv.indexOf("f")+1])) to_run.push({scriptPath : './unit_tests/add_rss.js', args : args});
});
run(to_run, "./results/rss.csv", function (err) {if (err) throw err;});
});
Milestone1/bench_files/run_scripts_sync.js
View file @ a6e13c59
/*jslint es6 */
"use strict";
function get_size(file) {
return file.slice(file.indexOf("_") + 1, file.lastIndexOf("."));
}
module.exports = {
run: function (args, filepath, callback) {
var ipc = require('node-ipc');
var ipc = require("node-ipc");
ipc.config.silent = true;
ipc.connectTo("main");
var childProcess = require('child_process');
const fs = require('fs');
var stream = fs.createWriteStream(filepath, {flags: 'a'});
var childProcess = require("child_process");
const fs = require("fs");
var stream = fs.createWriteStream(filepath, {flags: "a"});
function write(message) {
if (filepath === "") {
......@@ -23,7 +18,7 @@ module.exports = {
}
if (args.length === 0) {
ipc.of.main.emit('message', "DONE");
ipc.of.main.emit("message", "DONE");
return;
}
......@@ -32,29 +27,20 @@ module.exports = {
console.log(to_run.args);
var process = childProcess.fork(to_run.scriptPath, to_run.args);
process.on('message', function (mes) {
process.on("message", function (mes) {
mes = mes.toString();
if (mes.includes("DONE")) {
module.exports.run(args, filepath, callback);
} else if (mes.includes("ERROR")) {
ipc.of.main.emit('message', mes);
if (to_run.args.includes("-f")) {
write(get_size(to_run.args[1]) + "," + mes);
} else {
write(mes);
}
module.exports.run(args, filepath, callback);
} else {
ipc.of.main.emit('message', mes);
if (to_run.args.includes("-f")) {
write(get_size(to_run.args[1]) + "," + mes);
} else {
write(mes);
ipc.of.main.emit("message", mes);
write(mes);
if (mes.includes("ERROR")) {
module.exports.run(args, filepath, callback);
}
}
});
process.on('error', function (err) {
process.on("error", function (err) {
if (invoked) {
return;