added second script, randomly generates words, sentences and paragraphs to add...

added second script, randomly generates words, sentences and paragraphs to add to the db (1 par = 1 index entry)

load_index_random.js

+var os = require('os');
+var randomstring = require("randomstring");
+var tmp = require('tmp');
+var fs = require('fs');
+var args = require("yargs")
+.usage("Usage : load_index.js -saveAs metadata/attachment -p Number of random paragraphs to generate")
+.demandOption(['saveAs'])
+.demandOption(['paragraphs'])
+.alias("p", "paragraphs")
+.alias("s", "saveAs")
+.describe("paragraphs", "Number of random paragraphs to generate")
+.describe("saveAs", "choose whether to save the index as metadata or attachments in the indexeddb")
+.nargs("saveAs", 1)
+.nargs("paragraphs", 1)
+.argv;
+
+var gen_rdm_string = function (length){
+  return randomstring.generate({
+    length: length,
+    charset: 'alphabetic'
+  });
+}
+
+var gen_rdm_sentence = function (word_count, word_length){
+  var tmp_word_array = [];
+  for (var i = 0; i<word_count; i+=1){
+    tmp_word_array[i] = gen_rdm_string(word_length);
+  }
+  return tmp_word_array.join(" ");
+}
+
+var gen_rdm_paragraph = function (sentence_count, words_per_sentence, word_length){
+  var tmp_sentence_array = [];
+  for (var i=0; i<sentence_count; i+=1){
+    tmp_sentence_array[i] = gen_rdm_sentence(words_per_sentence, word_length);
+  }
+  return tmp_sentence_array.join(". ");
+}
+
+if (!(args.saveAs === "metadata" ||args.saveAs === "attachment")){
+  throw 'Unrecognized save as argument';
+}
+var saveAs = args.saveAs;
+
+const puppeteer = require('puppeteer');
+var browser, page, ramBefore, memBefore;
+
+const si = require("systeminformation");
+
+var Server = require('ws').Server;
+var port = 9030;
+var ws = new Server({port: port});
+ws.on('connection', function(w){
+  w.on('message', function(msg){
+    console.log(msg);
+    var ramAfter = (os.totalmem() - os.freemem()) / 1024 / 1024;
+    var ramUsed = ramAfter - ramBefore;
+    console.log("Ram used : " + ramUsed + " MB");
+    si.mem(function(data){
+      console.log("Memory used : " + (data.used / 1024 / 1024 - memBefore) + " MB");
+    });
+    browser.close();
+  });
+  w.on('close', function() {
+    ws.close();
+  });
+});
+
+(async () => {
+  browser = await puppeteer.launch();
+  page = await browser.newPage();
+  ramBefore = (os.totalmem() - os.freemem()) / 1024 / 1024;
+  si.mem(function(data){
+    memBefore = data.used / 1024 / 1024;
+  });
+
+  var tmp_para_array = [];
+  for(var i=0; i<args.p; i+=1){
+    tmp_para_array[i] = gen_rdm_paragraph(6, 15, 5);
+  }
+  var tmp_file = tmp.fileSync();
+  fs.writeFileSync(tmp_file.name, tmp_para_array.join("\n"));
+
+  await page.goto('https://softinst115787.host.vifib.net/public/unit_tests/index_load.html?saveAs='+saveAs);
+  const [fileChooser] = await Promise.all([
+    page.waitForFileChooser(),
+    page.click('input#load')
+  ]);
+  await fileChooser.accept([tmp_file.name]);
+})();

node_modules/.bin/randomstring

+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
+esac
+
+if [ -x "$basedir/node" ]; then
+  "$basedir/node"  "$basedir/../randomstring/bin/randomstring" "$@"
+  ret=$?
+else 
+  node  "$basedir/../randomstring/bin/randomstring" "$@"
+  ret=$?
+fi
+exit $ret

node_modules/.bin/randomstring.cmd

+@IF EXIST "%~dp0\node.exe" (
+  "%~dp0\node.exe"  "%~dp0\..\randomstring\bin\randomstring" %*
+) ELSE (
+  @SETLOCAL
+  @SET PATHEXT=%PATHEXT:;.JS;=;%
+  node  "%~dp0\..\randomstring\bin\randomstring" %*
+)
\ No newline at end of file

node_modules/array-uniq/index.js

+'use strict';
+
+// there's 3 implementations written in increasing order of efficiency
+
+// 1 - no Set type is defined
+function uniqNoSet(arr) {
+	var ret = [];
+
+	for (var i = 0; i < arr.length; i++) {
+		if (ret.indexOf(arr[i]) === -1) {
+			ret.push(arr[i]);
+		}
+	}
+
+	return ret;
+}
+
+// 2 - a simple Set type is defined
+function uniqSet(arr) {
+	var seen = new Set();
+	return arr.filter(function (el) {
+		if (!seen.has(el)) {
+			seen.add(el);
+			return true;
+		}
+	});
+}
+
+// 3 - a standard Set type is defined and it has a forEach method
+function uniqSetWithForEach(arr) {
+	var ret = [];
+
+	(new Set(arr)).forEach(function (el) {
+		ret.push(el);
+	});
+
+	return ret;
+}
+
+// V8 currently has a broken implementation
+// https://github.com/joyent/node/issues/8449
+function doesForEachActuallyWork() {
+	var ret = false;
+
+	(new Set([true])).forEach(function (el) {
+		ret = el;
+	});
+
+	return ret === true;
+}
+
+if ('Set' in global) {
+	if (typeof Set.prototype.forEach === 'function' && doesForEachActuallyWork()) {
+		module.exports = uniqSetWithForEach;
+	} else {
+		module.exports = uniqSet;
+	}
+} else {
+	module.exports = uniqNoSet;
+}

node_modules/array-uniq/package.json

+{
+  "_from": "array-uniq@1.0.2",
+  "_id": "array-uniq@1.0.2",
+  "_inBundle": false,
+  "_integrity": "sha1-X8w3OSB3VyPP1k1lxkvvU7+eum0=",
+  "_location": "/array-uniq",
+  "_phantomChildren": {},
+  "_requested": {
+    "type": "version",
+    "registry": true,
+    "raw": "array-uniq@1.0.2",
+    "name": "array-uniq",
+    "escapedName": "array-uniq",
+    "rawSpec": "1.0.2",
+    "saveSpec": null,
+    "fetchSpec": "1.0.2"
+  },
+  "_requiredBy": [
+    "/randomstring"
+  ],
+  "_resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.2.tgz",
+  "_shasum": "5fcc373920775723cfd64d65c64bef53bf9eba6d",
+  "_spec": "array-uniq@1.0.2",
+  "_where": "C:\\Users\\thequ\\Documents\\Mynij\\node_modules\\randomstring",
+  "author": {
+    "name": "Sindre Sorhus",
+    "email": "sindresorhus@gmail.com",
+    "url": "http://sindresorhus.com"
+  },
+  "bugs": {
+    "url": "https://github.com/sindresorhus/array-uniq/issues"
+  },
+  "bundleDependencies": false,
+  "deprecated": false,
+  "description": "Create an array without duplicates",
+  "devDependencies": {
+    "es6-set": "^0.1.0",
+    "mocha": "*",
+    "require-uncached": "^1.0.2"
+  },
+  "engines": {
+    "node": ">=0.10.0"
+  },
+  "files": [
+    "index.js"
+  ],
+  "homepage": "https://github.com/sindresorhus/array-uniq#readme",
+  "keywords": [
+    "array",
+    "arr",
+    "set",
+    "uniq",
+    "unique",
+    "es6",
+    "duplicate",
+    "remove"
+  ],
+  "license": "MIT",
+  "name": "array-uniq",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sindresorhus/array-uniq.git"
+  },
+  "scripts": {
+    "test": "mocha"
+  },
+  "version": "1.0.2"
+}

node_modules/array-uniq/readme.md

+# array-uniq [![Build Status](https://travis-ci.org/sindresorhus/array-uniq.svg?branch=master)](https://travis-ci.org/sindresorhus/array-uniq)
+
+> Create an array without duplicates
+
+It's already pretty fast, but will be much faster when [Set](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set) becomes available in V8 (especially with large arrays).
+
+
+## Install
+
+```sh
+$ npm install --save array-uniq
+```
+
+
+## Usage
+
+```js
+var arrayUniq = require('array-uniq');
+
+arrayUniq([1, 1, 2, 3, 3]);
+//=> [1, 2, 3]
+
+arrayUniq(['foo', 'foo', 'bar', 'foo']);
+//=> ['foo', 'bar']
+```
+
+
+## License
+
+MIT © [Sindre Sorhus](http://sindresorhus.com)

node_modules/randomstring/.npmignore

+.DS_Store
+*.log
+test.js
\ No newline at end of file

node_modules/randomstring/.travis.yml

+language: node_js
+node_js:
+  - "0.12"
+  - "0.11"
+  - "0.10"
+  - "iojs"
+  - "iojs-v1.0.4"
\ No newline at end of file

node_modules/randomstring/CHANGELOG.md

+1.1.5 / May 18, 2016
+==================
+  * Optimized character generation algorithm
+
+1.1.4 / Feb 10, 2016
+==================
+  * Added option for capitalization
+
+1.1.3 / Nov 03, 2015
+==================
+  * Fixed test
+
+1.1.2 / Nov 03, 2015
+==================
+  * Added command line support
+  * Fixed bug causing the "readable" option to fail
+
+1.1.0 / Sep 06, 2015
+==================
+  * Added support for custom character sets
+  * Added option for excluding poorly readable characters
+
+1.0.8 / Sep 01, 2015
+==================
+  * Avoid problems if crypto.randomBytes throws an exception
+
+1.0.7 / Jul 03, 2015
+==================
+  * Use node.crypto instead of Math.random as random number generator
+
+1.0.6 / Jun 01, 2015
+==================
+  * Added licence for npmjs.org
+  * Enhanced readme for Github and npm
+
+1.0.5 / Apr 03, 2015
+==================
+  * Better charset setting → Less error-proneness
+
+1.0.4 / Apr 03, 2015
+==================
+  * Added tests
+
+1.0.3 / Feb 17, 2014
+==================
+  * Fixed typo in character set
+
+1.0.0 / Jan 21, 2012
+==================
+  * Start of the project
\ No newline at end of file

node_modules/randomstring/LICENSE

+Copyright (c) 2012 Elias Klughammer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

node_modules/randomstring/README.md

+# node-randomstring
+
+[![Build Status](https://travis-ci.org/klughammer/node-randomstring.svg?branch=master)](https://travis-ci.org/klughammer/node-randomstring) [![Download Stats](https://img.shields.io/npm/dm/randomstring.svg)](https://github.com/klughammer/node-randomstring)
+
+Library to help you create random strings.
+
+## Installation
+
+To install randomstring, use [npm](http://github.com/npm/npm):
+
+```
+npm install randomstring
+```
+
+## Usage
+
+```javascript
+var randomstring = require("randomstring");
+
+randomstring.generate();
+// >> "XwPp9xazJ0ku5CZnlmgAx2Dld8SHkAeT"
+
+randomstring.generate(7);
+// >> "xqm5wXX"
+
+randomstring.generate({
+  length: 12,
+  charset: 'alphabetic'
+});
+// >> "AqoTIzKurxJi"
+
+randomstring.generate({
+  charset: 'abc'
+});
+// >> "accbaabbbbcccbccccaacacbbcbbcbbc"
+```
+
+## API
+
+`randomstring.`
+
+- `generate(options)`
+  - `length` - the length of the random string. (default: 32) [OPTIONAL]
+  - `readable` - exclude poorly readable chars: 0OIl. (default: false) [OPTIONAL]
+  - `charset` - define the character set for the string. (default: 'alphanumeric') [OPTIONAL]
+    - `alphanumeric` - [0-9 a-z A-Z]
+    - `alphabetic` - [a-z A-Z]
+    - `numeric` - [0-9]
+    - `hex` - [0-9 a-f]
+    - `custom` - any given characters
+  - `capitalization` - define whether the output should be lowercase / uppercase only. (default: null) [OPTIONAL]
+    - `lowercase`
+    - `uppercase`
+
+## Command Line Usage
+
+    $ npm install -g randomstring
+
+    $ randomstring
+    > sKCx49VgtHZ59bJOTLcU0Gr06ogUnDJi
+
+    $ randomstring 7
+    > CpMg433
+    
+    $ randomstring length=24 charset=github readable
+    > hthbtgiguihgbuttuutubugg
+
+## Tests
+
+```
+npm install
+npm test
+```
+
+## LICENSE
+
+node-randomstring is licensed under the MIT license.

node_modules/randomstring/bin/randomstring

+#!/usr/bin/env node
+
+var randomstring = require('..');
+
+var options = {};
+
+for (var i = 2, ii = process.argv.length; i < ii; i++) {
+  var arg = process.argv[i];
+  
+  if (arg.search('length') >= 0) {
+    var length = arg.split('=')[1];
+    
+    if (length)
+      options.length = parseInt(length);
+  }
+  else if (arg.search('readable') >= 0) {
+    options.readable = true;
+  }
+  else if (arg.search('charset') >= 0) {
+    var charset = arg.split('=')[1];
+    
+    if (charset)
+      options.charset = charset;
+  }
+  else if (arg.search('capitalization') >= 0) {
+    var capitalization = arg.split('=')[1];
+    
+    if (capitalization)
+      options.capitalization = capitalization;
+  }
+  // If only a number is given as an arg, parse it as the length
+  else if (arg.search(/\D/ig) < 0) {
+    options.length = parseInt(arg);
+  }
+};
+
+var result = randomstring.generate(options);
+
+console.log(result)
\ No newline at end of file

node_modules/randomstring/index.js

+module.exports = require("./lib/randomstring");
\ No newline at end of file

node_modules/randomstring/lib/charset.js

+var arrayUniq = require('array-uniq');
+
+function Charset() {
+  this.chars = '';
+}
+
+Charset.prototype.setType = function(type) {
+  var chars;
+  
+  var numbers    = '0123456789';
+  var charsLower = 'abcdefghijklmnopqrstuvwxyz';
+  var charsUpper = charsLower.toUpperCase();
+  var hexChars   = 'abcdef';
+  
+  if (type === 'alphanumeric') {
+    chars = numbers + charsLower + charsUpper;
+  }
+  else if (type === 'numeric') {
+    chars = numbers;
+  }
+  else if (type === 'alphabetic') {
+    chars = charsLower + charsUpper;
+  }
+  else if (type === 'hex') {
+    chars = numbers + hexChars;
+  }
+  else {
+    chars = type;
+  }
+  
+  this.chars = chars;
+}
+
+Charset.prototype.removeUnreadable = function() {
+  var unreadableChars = /[0OIl]/g;
+  this.chars = this.chars.replace(unreadableChars, '');
+}
+
+Charset.prototype.setcapitalization = function(capitalization) {
+  if (capitalization === 'uppercase') {
+    this.chars = this.chars.toUpperCase();
+  }
+  else if (capitalization === 'lowercase') {
+    this.chars = this.chars.toLowerCase();
+  }
+}
+
+Charset.prototype.removeDuplicates = function() {
+  var charMap = this.chars.split('');
+  charMap = arrayUniq(charMap);
+  this.chars = charMap.join('');
+}
+
+module.exports = exports = Charset;
\ No newline at end of file

node_modules/randomstring/lib/randomstring.js

+"use strict";
+
+var crypto  = require('crypto');
+var Charset = require('./charset.js');
+
+function safeRandomBytes(length) {
+  while (true) {
+    try {
+      return crypto.randomBytes(length);
+    } catch(e) {
+      continue;
+    }
+  }
+}
+
+exports.generate = function(options) {
+  
+  var charset = new Charset();
+  
+  var length, chars, capitalization, string = '';
+  
+  // Handle options
+  if (typeof options === 'object') {
+    length = options.length || 32;
+    
+    if (options.charset) {
+      charset.setType(options.charset);
+    }
+    else {
+      charset.setType('alphanumeric');
+    }
+    
+    if (options.capitalization) {
+      charset.setcapitalization(options.capitalization);
+    }
+    
+    if (options.readable) {
+      charset.removeUnreadable();
+    }
+    
+    charset.removeDuplicates();
+  }
+  else if (typeof options === 'number') {
+    length = options;
+    charset.setType('alphanumeric');
+  }
+  else {
+    length = 32;
+    charset.setType('alphanumeric');
+  }
+  
+  // Generate the string
+  var charsLen = charset.chars.length;
+  var maxByte = 256 - (256 % charsLen);
+  while (length > 0) {
+    var buf = safeRandomBytes(Math.ceil(length * 256 / maxByte));
+    for (var i = 0; i < buf.length && length > 0; i++) {
+      var randomByte = buf.readUInt8(i);
+      if (randomByte < maxByte) {
+        string += charset.chars.charAt(randomByte % charsLen);
+        length--;
+      }
+    }
+  }
+
+  return string;
+};

node_modules/randomstring/package.json

+{
+  "_from": "randomstring",
+  "_id": "randomstring@1.1.5",
+  "_inBundle": false,
+  "_integrity": "sha1-bfBij3XL1ZMpMNn+OrTpVqGFGMM=",
+  "_location": "/randomstring",
+  "_phantomChildren": {},
+  "_requested": {
+    "type": "tag",
+    "registry": true,
+    "raw": "randomstring",
+    "name": "randomstring",
+    "escapedName": "randomstring",
+    "rawSpec": "",
+    "saveSpec": null,
+    "fetchSpec": "latest"
+  },
+  "_requiredBy": [
+    "#USER",
+    "/"
+  ],
+  "_resolved": "https://registry.npmjs.org/randomstring/-/randomstring-1.1.5.tgz",
+  "_shasum": "6df0628f75cbd5932930d9fe3ab4e956a18518c3",
+  "_spec": "randomstring",
+  "_where": "C:\\Users\\thequ\\Documents\\Mynij",
+  "author": {
+    "name": "Elias Klughammer",
+    "email": "elias@klughammer.com",
+    "url": "http://www.klughammer.com"
+  },
+  "bin": {
+    "randomstring": "bin/randomstring"
+  },
+  "bugs": {
+    "url": "https://github.com/klughammer/node-randomstring/issues"
+  },
+  "bundleDependencies": false,
+  "dependencies": {
+    "array-uniq": "1.0.2"
+  },
+  "deprecated": false,
+  "description": "A module for generating random strings",
+  "devDependencies": {
+    "mocha": "^1.20.1"
+  },
+  "engines": {
+    "node": "*"
+  },
+  "homepage": "https://github.com/klughammer/node-randomstring",
+  "license": "MIT",
+  "main": "./index",
+  "name": "randomstring",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/klughammer/node-randomstring.git"
+  },
+  "scripts": {
+    "test": "mocha"
+  },
+  "version": "1.1.5"
+}

node_modules/randomstring/test/index.js

+"use strict";
+
+var assert = require("assert");
+var random = require("..").generate;
+var testLength = 24700;
+
+describe("randomstring.generate(options)", function() {
+
+  it("returns a string", function() {
+    var rds = random();
+    assert.equal(typeof(rds), "string");
+  });
+
+  it("defaults to 32 characters in length", function() {
+    assert.equal(random().length, 32);
+  });
+
+  it("accepts length as an optional first argument", function() {
+    assert.equal(random(10).length, 10);
+  });
+  
+  it("accepts length as an option param", function() {
+    assert.equal(random({ length: 7 }).length, 7);
+  });
+  
+  it("accepts 'numeric' as charset option", function() {
+    var testData = random({ length: testLength, charset: 'numeric' });
+    var search = testData.search(/\D/ig);
+    assert.equal(search, -1);
+  });