WIP: jIO storage to provide search indexing for storages without search functionality

test/jio.storage/liststorage.tests.js

+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+/*jslint nomen: true */
+/*global Blob, indexedDB*/

test/jio.storage/liststorage.tests.js

+      sub_storage: {
+        type: "dummystorage1"
+      }
+    });
+
+    DummyStorage.prototype.post = function (param) {
+      deepEqual(param, {"name": "test_name"});
+      return "posted";
+    };
+
+    jio.post({"name": "test_name"})
+      .then(function (result) {
+        equal(result, "posted");
+        jio.__storage._signature_storage.get("posted")
+          .then(function (result) {
+            equal(result.id, "posted");

test/jio.storage/indexstorage2.tests.js

+  }
+  jIO.addStorage('dummystorage3', DummyStorage3);
+
+  /////////////////////////////////////////////////////////////////
+  // indexStorage2.constructor
+  /////////////////////////////////////////////////////////////////
+  module("indexStorage2.constructor", {
+    teardown: function () {
+      deleteIndexedDB(this.jio);
+    }
+  });
+  test("Constructor with empty index_keys", function () {
+    this.jio = jIO.createJIO({
+      type: "index2",
+      database: "index2_test",
+      index_keys: [],

test/jio.storage/indexstorage2.tests.js

+    teardown: function () {
+      deleteIndexedDB(this.jio);
+    }
+  });
+  test("can list documents", function () {
+    this.jio = jIO.createJIO({
+      type: "index2",
+      database: "index2_test",
+      index_keys: [],
+      sub_storage: {
+        type: "dummystorage3"
+      }
+    });
+
+    ok(this.jio.hasCapacity("list"));
+    ok(this.jio.hasCapacity("query"));

test/jio.storage/indexstorage2.tests.js

+    var context = this;
+    stop();
+    expect(4);
+
+    DummyStorage3.prototype.put = function (id, value) {
+      equal(id, "32");
+      deepEqual(value, {"a": 3, "b": 2, "c": 8});
+      return id;
+    };
+
+    DummyStorage3.prototype.get = function (id) {
+      equal(id, "32");
+      return {"a": 3, "b": 2, "c": 8};
+    };
+
+    context.jio.put("32", {"a": 3, "b": 2, "c": 8})

test/jio.storage/indexstorage2.tests.js

+      equal(id, "32");
+      return {"a": 3, "b": 2, "c": 8};
+    };
+
+    context.jio.put("32", {"a": 3, "b": 2, "c": 8})
+      .then(function () {
+        return context.jio.get("32");
+      })
+      .then(function (result) {
+        deepEqual(result, {"a": 3, "b": 2, "c": 8});
+      })
+      .fail(function (error) {
+        console.log(error);
+      })
+      .then(function () {
+        return deleteIndexedDB(context.jio);

test/jio.storage/indexstorage2.tests.js

+    });
+    stop();
+    expect(3);
+
+    DummyStorage3.prototype.put = function (id, value) {
+      equal(id, "32");
+      deepEqual(value, {a: "3", b: "2"});
+      return id;
+    };
+
+    context.jio.put("32", {"a": "3", "b": "2"})
+      .then(function () {
+        return context.jio.allDocs({query: 'a: "3"'});
+      })
+      .then(function (result) {
+        deepEqual(result.data.rows[0], {"id": "32", "value": {}});

test/jio.storage/indexstorage2.tests.js

+      type: "index2",
+      database: "index2_test",
+      index_keys: [],
+      sub_storage: {
+        type: "dummystorage3"
+      }
+    });
+    stop();
+    expect(4);
+
+    DummyStorage3.prototype.put = function (id) {
+      return id;
+    };
+    DummyStorage3.prototype.hasCapacity = function (capacity) {
+      equal(capacity, "query");
+      if (capacity === "query") { return true; }

test/jio.storage/indexstorage2.tests.js

+    expect(1);
+
+    DummyStorage3.prototype.put = function (id) {
+      return id;
+    };
+
+    RSVP.all([
+      context.jio.put("32", {"a": "3", "b": "2"}),
+      context.jio.put("21", {"a": "6", "b": "9"}),
+      context.jio.put("3", {"a": "8", "b": "5"})
+    ])
+      .then(function () {
+        return context.jio.allDocs();
+      })
+      .then(function (result) {
+        equal(result.data.total_rows, 3);

src/jio.storage/indexstorage2.js

+  };
+
+  function handleUpgradeNeeded(evt, index_keys) {
+    var db = evt.target.result, store, i;
+
+    store = db.createObjectStore("index-store", {
+      keyPath: "id",
+      autoIncrement: false
+    });
+    for (i = 0; i < index_keys.length; i += 1) {
+      store.createIndex("Index-" + index_keys[i], "doc." + index_keys[i],
+        {unique: false});
+    }
+  }
+
+  function waitForOpenIndexedDB(db_name, index_keys, callback) {

src/jio.storage/indexstorage2.js

+    return new RSVP.Promise(function (resolve, reject) {
+      request.onerror = reject;
+      request.onsuccess = resolve;
+    });
+  }
+
+  IndexStorage2.prototype._runQuery = function (index, value) {
+    var context = this;
+    return new RSVP.Queue()
+      .push(function () {
+        if ((context._index_keys.indexOf(index) === -1)) {
+          if (context._sub_storage.hasCapacity("query")) {
+            return context._sub_storage.buildQuery(
+              {"query": index + ":" + value}
+            )
+              .then(function (result) {

src/jio.storage/indexstorage2.js

+    return filtered_doc;
+  };
+
+  IndexStorage2.prototype.put = function (id, value) {
+    var context = this;
+    return context._sub_storage.put(id, value)
+      .push(function (result) {
+        return waitForOpenIndexedDB(context._database_name,
+          context._index_keys, function (db) {
+            return waitForTransaction(db, ["index-store"], "readwrite",
+                                function (tx) {
+                return waitForIDBRequest(tx.objectStore("index-store").put({
+                  "id": id,
+                  "doc": context._filter_doc_values(value, context._index_keys)
+                }))
+                  .then(function () {

src/jio.storage/indexstorage2.js

+          function (tx) {
+            return waitForIDBRequest(tx.objectStore("index-store").getAll())
+              .then(function (evt) {
+                return evt.target.result.map(function (value) {
+                  return {"id": value.id, "value": {} };
+                });
+              });
+          });
+      });
+  };
+
+  IndexStorage2.prototype.get = function () {
+    return this._sub_storage.get.apply(this._sub_storage, arguments);
+  };
+
+  IndexStorage2.prototype._filter_doc_values = function (doc, keys) {

src/jio.storage/indexstorage2.js

+                  }
+                  query_result = temp_set;
+                  temp_set = new Set();
+                }
+                return Array.from(query_result);
+              }
+            });
+        }
+      });
+  };
+
+  IndexStorage2.prototype.buildQuery = function (options) {
+    var context = this;
+    if (options.query) {
+      return this._processQueryObject(parseStringToObject(options.query))
+        .then(function (result) {

src/jio.storage/indexstorage2.js

+    this._database_name = "jio:" + description.database;
+    this._index_keys = description.index_keys;
+  }
+
+  IndexStorage2.prototype.hasCapacity = function (name) {
+    return ((name === "list") || (name === "query"));
+  };
+
+  function handleUpgradeNeeded(evt, index_keys) {
+    var db = evt.target.result, store, i;
+
+    store = db.createObjectStore("index-store", {
+      keyPath: "id",
+      autoIncrement: false
+    });
+    for (i = 0; i < index_keys.length; i += 1) {

@preet please do not use indexedDB in the test. Instead, create dummy storage which will ensure you that their methods are correctly called with the expected parameters.

@preet if you forget to check (ie, use 'return') a promise call result, you will not know when it is finished or when it fails. In this case, you can't be sure that equal will be executed before the stop method

@preet do you store the index_keys somewhere? If yes, please test.

@preet what is the status of the other capacity supported by querystorage?

@preet is it required to call put? probably not, as you're mocking the substorage get call.

@preet don't you already delete during teardown?

@preet can you test the full result value? because currently, it can return many values

@preet return (capacity === 'query')

@preet why does it work, if the substorage does not do the query?

If no query is used, buildQuery returns all of the keys from the object store which contains the indices. The object store uses the keys("32", "21", "3" here) as the primary key. Therefore the sub-storage isn't called in this case for the query.

@preet please try not to duplicate the indexstorage code. Can you use a common code for both?

@preet not needed

@preet not needed

@preet please follow naming convention

@preet in jIO, .then usage is not accepted (or, you must explain why it is enough). Please use Queue.push instead

@preet how will the migration be handled? (ie, the list of keys are modified in the jio configuration parameters)