IndexStorage initial commit

4c8d5307 · preetwinder · Romain Courteaud · 9d9506b7 · 4c8d5307 · 4c8d5307
Commit 4c8d5307 authored Feb 28, 2019 by preetwinder Committed by Romain Courteaud Apr 19, 2019
7 changed files
--- a/Makefile
+++ b/Makefile
@@ -153,7 +153,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
 	${SRCDIR}/jio.storage/fbstorage.js \
 	${SRCDIR}/jio.storage/cloudooostorage.js \
 	${SRCDIR}/jio.storage/nocapacitystorage.js \
-	${SRCDIR}/jio.storage/liststorage.js
+	${SRCDIR}/jio.storage/liststorage.js \
+	${SRCDIR}/jio.storage/indexstorage2.js
 	@mkdir -p $(@D)
 	cat $^ > $@

--- a/examples/index2_benchmark.html
+++ b/examples/index2_benchmark.html
+<!DOCTYPE html>
+<!--
+Copyright 2019, Nexedi SA
+This program is free software: you can Use, Study, Modify and Redistribute
+it under the terms of the GNU General Public License version 3, or (at your
+option) any later version, as published by the Free Software Foundation.
+You can also Link and Combine this program with other software covered by
+the terms of any of the Free Software licenses or any of the Open Source
+Initiative approved licenses and Convey the resulting work. Corresponding
+source of such a combination shall include the source code for all other
+software used.
+This program is distributed WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See COPYING file for full licensing terms.
+See https://www.nexedi.com/licensing for rationale and options.
+-->
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>jIO Query Performance test</title>
+  <script src="../external/rsvp-2.0.4.js"></script>
+  <script src="../external/renderjs-latest.js"></script>
+  <script src="../dist/jio-latest.js"></script>
+  <!-- <link rel="stylesheet" href="../external/qunit.css" type="text/css" media="screen"/> -->
+  <!-- <script src="../external/qunit.js" type="text/javascript"></script> -->
+  <script src="index2_benchmark.js"></script>
+</head>
+<body>
+  <h1> Testing index2 query </h1>
+  <a>Test</a>
+  <p></p>
+  <div></div>
+</body>
+</html>
--- a/examples/index2_benchmark.js
+++ b/examples/index2_benchmark.js
+/*global performance, String*/
+(function (window, jIO, rJS) {
+  "use strict";
+  var test_count = 15;
+  /*function randomi(limit) {
+    return Math.floor(Math.random() * Math.floor(limit));
+  }
+  function randomSentence(length) {
+    var alphabet = ['a', 'b', 'c', 'd', 'e', ' ', 'f', 'g', 'h', 'i', 'j', 'k',
+      ' ', 'l', 'm', 'n', 'o', ' ', 'p', 'q', 'r', 's', 't', ' ',
+      'u', 'v', 'w', ' ', 'x', 'y', 'z', ' '], sentence = '', z;
+    for (z = 0; z < length; z += 1) {
+      sentence += alphabet[randomi(alphabet.length - 1)];
+    }
+    return sentence;
+  }
+  function randomSentenceArray(sentence_length, array_length) {
+    var y, sentence_array = [];
+    for (y = 0; y < array_length; y += 1) {
+      sentence_array.push(randomSentence(sentence_length));
+    }
+    return sentence_array;
+  }*/
+  function get_fake_data_values2(i) {
+    if (i === 0 || i === 1 || i === 2) {
+      return {'url': 'renderjs.com', 'name': 'erp5', 'user': 'preet'};
+    }
+    if (i === 3 || i === 4) {
+      return {'url': 'erp5.com', 'name': 'erp5', 'user': 'test'};
+    }
+    if (i === 5 || i === 6 || i === 7) {
+      return {'url': 'nexedi.com', 'name': 'nexedi', 'user': 'prabetcder'};
+    }
+    if (i === 10 || i === 11) {
+      return {'url': 'vifib.com', 'name': 'renderjs', 'user': 'preetwinder'};
+    }
+    if (i === 12 || i === 13) {
+      return {'url': 'renderjs.com', 'name': 'jio', 'user': 'obscure'};
+    }
+    return {'url': 'jio.nexedi.com', 'name': 'jio', 'user': 'praounsteter'};
+  }
+  /*function get_fake_data_values(i) {
+    var data_value = {
+      'id': i,
+      'url': 'https://streetsite.com/profiles/' + i,
+      'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i,
+      'short_description': randomSentence(10 + randomi(40)),
+      'description': randomSentence(randomi(250)),
+      'comments': randomSentenceArray(randomi(500), randomi(20))
+    };
+    if (i === 9900) {
+      data_value.short_description = 'test';
+    }
+    if (i === 7500) {
+      data_value.short_description = 'preet';
+    }
+    if (i === 5400) {
+      data_value.short_description = 'obscure';
+    }
+    if (i === 3200) {
+      data_value.short_description = 'precise';
+    }
+    if (i === 1200) {
+      data_value.short_description = 'environ';
+    }
+    return data_value;
+  }*/
+ /* function sequential_test(i, storage) {
+    if (i < test_count) {
+      var data_value = {
+        'id': i,
+        'url': 'https://streetsite.com/profiles/' + i,
+        'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i,
+        'short_description': randomSentence(10 + randomi(40)),
+        'description': randomSentence(randomi(250)),
+        'comments': randomSentenceArray(randomi(500), randomi(20))
+      };
+      if (i === 99000) {
+        data_value.short_description = 'test';
+      }
+      if (i % 100 === 0) {
+        data_value.short_description = 'preet';
+      }
+      if (i % 1000 === 0) {
+        data_value.short_description = 'obscure';
+      }
+      if (i === 32000) {
+        data_value.short_description = 'precise precise precise';
+      }
+      if (i === 120000) {
+        data_value.short_description = 'environ';
+      }
+      return storage.put(String(i), data_value)
+        .then(function () {
+          if (i % 1000 === 0) {
+            console.log(i);
+          }
+          data_value = null;
+          return sequential_test(i + 1, storage);
+        });
+    }
+    return;
+  }*/
+  rJS(window)
+    .declareService(function () {
+      var storage = jIO.createJIO({
+        type: "index2",
+        database: "index2test2",
+        index_keys: ["user", "name", "url"],
+        sub_storage: {
+          type: "indexeddb",
+          database: "index2testdata2",
+        }
+      }), promise_list = [], i, time;
+      console.log('Staring to write ' + test_count + ' documents');
+      //sequential_test(0, storage);
+      for (i = 0; i < test_count; i += 1) {
+        promise_list.push(storage.put(String(i), get_fake_data_values2(i)));
+      }
+      promise_list.push(storage.put('325', get_fake_data_values2(325)));
+      time = performance.now();
+      return RSVP.all(promise_list)
+        .then(function () {
+          console.log('Time to write - ', (performance.now() - time));
+          console.log('Starting queries');
+          console.log('Query 1');
+          time = performance.now();
+          return storage.allDocs({query: "user:preetwinder"});
+        })
+        .then(function (result) {
+          console.log('Time to query 1 - ', (performance.now() - time));
+          console.log(result);
+          console.log('Query 2');
+          time = performance.now();
+          return storage.allDocs({query: 'user:preet'});
+        })
+        .then(function (result) {
+          console.log('Time to query 2 - ', (performance.now() - time));
+          console.log(result);
+          console.log('Query 3');
+          time = performance.now();
+          return storage.allDocs({query: "(name:jio OR url:nexedi.com" +
+            ") AND user:obscure"});
+        })
+        .then(function (result) {
+          console.log('Time to query 3 - ', performance.now() - time);
+          console.log(result);
+          console.log('Query 4');
+          time = performance.now();
+          return storage.allDocs({query: 'name:not'});
+        })
+        .then(function (result) {
+          console.log('Time to query 4 - ', performance.now() - time);
+          console.log(result);
+        });
+    });
+}(window, jIO, rJS));
\ No newline at end of file
--- a/src/jio.storage/indexstorage2.js
+++ b/src/jio.storage/indexstorage2.js
--- a/test/jio.storage/indexstorage2.tests.js
+++ b/test/jio.storage/indexstorage2.tests.js
--- a/test/node.js
+++ b/test/node.js
@@ -64,6 +64,7 @@
      'test/jio.storage/erp5storage.tests.js',
      'test/jio.storage/fbstorage.tests.js',
      'test/jio.storage/gdrivestorage.tests.js',
+      'test/jio.storage/indexstorage2.tests.js',
      'test/jio.storage/liststorage.tests.js',
      'test/jio.storage/memorystorage.tests.js',
      'test/jio.storage/nocapacitystorage.tests.js',

--- a/test/tests.html
+++ b/test/tests.html
@@ -79,6 +79,7 @@ See https://www.nexedi.com/licensing for rationale and options.
  <script src="jio.storage/httpstorage.tests.js"></script>
  <script src="jio.storage/nocapacitystorage.tests.js"></script>
  <script src="jio.storage/liststorage.tests.js"></script>
+  <script src="jio.storage/indexstorage2.tests.js"></script>
  <!--script src="../src/jio.storage/xwikistorage.js"></script>
  <script src="jio.storage/xwikistorage.tests.js"></script-->