feat(storage): add sql query storage

.gitignore

@@ -5,4 +5,6 @@
 #dolphin
 .directory
 
-node_modules/*
\ No newline at end of file
+node_modules/*
+
+package-lock.json

Gruntfile.js

@@ -182,7 +182,8 @@ module.exports = function (grunt) {
           'src/jio.storage/indexeddbstorage.js',
           'src/jio.storage/cryptstorage.js',
           'src/jio.storage/websqlstorage.js',
-          'src/jio.storage/fbstorage.js'
+          'src/jio.storage/fbstorage.js',
+          'src/jio.storage/sqlstorage.js'
         ],
         dest: 'dist/<%= pkg.name %>-<%= pkg.version %>.js'
 //         dest: 'jio.js'

src/jio.storage/sqlstorage.js

+/*
+ * Copyright 2013, Nexedi SA
+ * Released under the LGPL license.
+ * http://www.gnu.org/licenses/lgpl.html
+ */
+/**
+ * JIO Sql Storage. Type = "sql".
+ * sql "database" storage.
+ */
+/*global Blob, jIO, RSVP, SQL, localStorage*/
+/*jslint nomen: true*/
+
+(function (jIO, RSVP, SQL, localStorage) {
+  "use strict";
+
+  var sqlStorageKey = 'jio_sql',
+    sqlTable = "jiosearch";
+
+  function resetDb(indexFields) {
+    var db = new SQL.Database(),
+      fields = ["id"].concat(indexFields);
+    db.run("CREATE TABLE " + sqlTable + " ( " + fields.join(", ") + ");");
+    return db;
+  }
+
+  function initDb(indexFields) {
+    var data = localStorage.getItem(sqlStorageKey);
+    if (data) {
+      return new SQL.Database(data.split(","));
+    }
+
+    return resetDb(indexFields);
+  }
+
+  function saveDb(db) {
+    var data = db["export"](); // jslint throws error
+    localStorage.setItem(sqlStorageKey, data);
+  }
+
+  function docToParams(id, doc) {
+    var data = {};
+    Object.keys(doc).forEach(function (key) {
+      data[":" + key] = doc[key];
+    });
+    data[":id"] = id.toString();
+    return data;
+  }
+
+  function dbValues(indexFields) {
+    return " VALUES (:id, " + indexFields.map(function (field) {
+      return ":" + field;
+    }).join(", ") + ")";
+  }
+
+  function dbSet(indexFields) {
+    return " SET " + indexFields.map(function (field) {
+      return field + " = :" + field;
+    }).join(", ");
+  }
+
+  function dbOrderBy(fields) {
+    return (fields || []).map(function (field) {
+      return field[0] + " " + (field[1] === "descending" ? "DESC" : "ASC");
+    }).join(", ");
+  }
+
+  function dbWhere(fields, operator) {
+    var where = "",
+      field,
+      i;
+    for (i = 0; i < fields.length; i = i + 1) {
+      field = fields[i];
+      where += field.key + " LIKE \"%" + field.value + "%\"";
+      if (i !== fields.length - 1) {
+        where += " " + (field.operator || operator) + " ";
+      }
+    }
+    return where;
+  }
+
+  function filterJSON(json, fields) {
+    if (!fields || !fields.length) {
+      return json;
+    }
+    var data = {};
+    fields.forEach(function (field) {
+      if (json.hasOwnProperty(field)) {
+        data[field] = json[field];
+      }
+    });
+    return data;
+  }
+
+  /**
+    * The jIO sql.js extension
+    *
+    * @class SqlStorage
+    * @constructor
+    */
+  function SqlStorage(spec) {
+    this._sub_storage = jIO.createJIO(spec.sub_storage);
+    this.__index_fields = spec.index_fields;
+    this.__db = initDb(spec.index_fields || []);
+  }
+
+  SqlStorage.prototype.__resetDb = function (indexFields) {
+    this.__index_fields = indexFields;
+    this.__db = resetDb(indexFields);
+  };
+
+  SqlStorage.prototype.get = function () {
+    return this._sub_storage.get.apply(this._sub_storage, arguments);
+  };
+
+  SqlStorage.prototype.allAttachments = function () {
+    return this._sub_storage.allAttachments.apply(this._sub_storage, arguments);
+  };
+
+  SqlStorage.prototype.post = function (doc) {
+    var db = this.__db,
+      indexFields = this.__index_fields;
+
+    return this._sub_storage.post.apply(this._sub_storage, arguments)
+      .push(function (id) {
+        db.run(
+          "INSERT INTO " + sqlTable + dbValues(indexFields),
+          docToParams(id, doc)
+        );
+        saveDb(db);
+      });
+  };
+
+  SqlStorage.prototype.put = function (id, doc) {
+    var db = this.__db,
+      indexFields = this.__index_fields;
+
+    return this._sub_storage.put.apply(this._sub_storage, arguments)
+      .push(function () {
+        db.run(
+          "UPDATE " + sqlTable + dbSet(indexFields) + " WHERE id=:id",
+          docToParams(id, doc)
+        );
+        saveDb(db);
+      });
+  };
+
+  SqlStorage.prototype.remove = function (id) {
+    var db = this.__db;
+
+    return this._sub_storage.remove(id)
+      .push(function () {
+        db.run("DELETE FROM " + sqlTable + " WHERE id=:id", {
+          ":id": id
+        });
+        saveDb(db);
+      });
+  };
+
+  SqlStorage.prototype.getAttachment = function () {
+    return this._sub_storage.getAttachment.apply(this._sub_storage, arguments);
+  };
+
+  SqlStorage.prototype.putAttachment = function () {
+    return this._sub_storage.putAttachment.apply(this._sub_storage, arguments);
+  };
+
+  SqlStorage.prototype.removeAttachment = function () {
+    return this._sub_storage.removeAttachment.apply(this._sub_storage,
+                                                    arguments);
+  };
+
+  SqlStorage.prototype.repair = function () {
+    // rebuild db?
+    return this._sub_storage.repair.apply(this._sub_storage, arguments);
+  };
+
+  SqlStorage.prototype.hasCapacity = function (name) {
+    var this_storage_capacity_list = ["limit",
+                                      "sort",
+                                      "select",
+                                      "query"];
+
+    if (this_storage_capacity_list.indexOf(name) !== -1) {
+      return true;
+    }
+    if (name === "list") {
+      return this._sub_storage.hasCapacity(name);
+    }
+    return false;
+  };
+
+  SqlStorage.prototype.buildQuery = function (options) {
+    if (!options.query) {
+      return this._sub_storage.buildQuery(options);
+    }
+
+    var context = this,
+      db = this.__db,
+      parsed_query = jIO.QueryFactory.create(options.query);
+
+    return new RSVP.Queue()
+      .push(function () {
+        var query = "SELECT id FROM " + sqlTable,
+          where = dbWhere(
+            parsed_query.query_list || [parsed_query],
+            parsed_query.operator
+          ),
+          orderBy = dbOrderBy(options.sort_on),
+          limit = options.limit;
+
+        if (where) {
+          query += " WHERE " + where;
+        }
+
+        if (orderBy) {
+          query += " ORDER BY " + orderBy;
+        }
+
+        if (limit) {
+          query += " LIMIT " + limit.join(", ");
+        }
+
+        return db.prepare(query);
+      })
+      .push(function (result) {
+        var ids = [];
+        while (result.step()) {
+          ids.push(result.get()[0]);
+        }
+        result.free();
+
+        return RSVP.all(ids.map(function (id) {
+          return context.get(id).push(function (doc) {
+            return {
+              id: id,
+              value: filterJSON(doc, options.select_list)
+            };
+          });
+        }));
+      });
+  };
+
+  jIO.addStorage('sql', SqlStorage);
+
+}(jIO, RSVP, SQL, localStorage));

test/jio.storage/sqlstorage.tests.js

+/*jslint nomen: true */
+/*global jIO, QUnit, sinon, localStorage */
+(function (jIO, QUnit, sinon, localStorage) {
+  "use strict";
+  var test = QUnit.test,
+    stop = QUnit.stop,
+    start = QUnit.start,
+    ok = QUnit.ok,
+    deepEqual = QUnit.deepEqual,
+    equal = QUnit.equal,
+    module = QUnit.module;
+
+  /////////////////////////////////////////////////////////////////
+  // Custom test substorage definition
+  /////////////////////////////////////////////////////////////////
+
+  function Storage200() {
+    this.documents = {};
+    return this;
+  }
+
+  Storage200.prototype.get = function (id) {
+    var context = this;
+    return new RSVP.Queue().push(function () {
+      return context.documents[id];
+    });
+  };
+
+  Storage200.prototype.post = function (data) {
+    var context = this;
+    return new RSVP.Queue().push(function () {
+      var id = (Object.keys(context.documents).length + 1).toString();
+      context.documents[id] = data;
+      return id;
+    });
+  };
+
+  Storage200.prototype.put = function (id, data) {
+    var context = this;
+    return new RSVP.Queue().push(function () {
+      context.documents[id] = data;
+      return context.documents[id];
+    });
+  };
+
+  Storage200.prototype.remove = function (id) {
+    delete this.documents[id];
+    return new RSVP.Queue();
+  };
+
+  Storage200.prototype.hasCapacity = function () {
+    return true;
+  };
+
+  Storage200.prototype.buildQuery = function () {
+    var context = this;
+    return new RSVP.Queue().push(function () {
+      return Object.keys(context.documents).map(function (id) {
+        return {
+          id: id,
+          value: context.documents[id]
+        };
+      });
+    });
+  };
+
+  jIO.addStorage('sql200', Storage200);
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.constructor
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.constructor", {
+    setup: function () {
+      this.getItemSpy = sinon.spy(localStorage, "getItem");
+
+      this.jio = jIO.createJIO({
+        type: "sql",
+        sub_storage: {
+          type: "sql200"
+        }
+      });
+    },
+    teardown: function () {
+      this.getItemSpy.restore();
+      delete this.getItemSpy;
+    }
+  });
+
+  test("set the type", function () {
+    equal(this.jio.__type, "sql");
+  });
+
+  test("spy load from localStorage", function () {
+    ok(
+      this.getItemSpy.calledOnce,
+      "getItem count " + this.getItemSpy.callCount
+    );
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.hasCapacity
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.hasCapacity");
+  test("can list documents", function () {
+    var jio = jIO.createJIO({
+      type: "sql",
+      sub_storage: {
+        type: "sql200"
+      }
+    });
+
+    ok(jio.hasCapacity("list"));
+  });
+
+  test("can query documents", function () {
+    var jio = jIO.createJIO({
+      type: "sql",
+      sub_storage: {
+        type: "sql200"
+      }
+    });
+
+    ok(jio.hasCapacity("query"));
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.post
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.post", {
+    setup: function () {
+      this.jio = jIO.createJIO({
+        type: "sql",
+        index_fields: [
+          "title"
+        ],
+        sub_storage: {
+          type: "sql200"
+        }
+      });
+      this.jio.__storage.__resetDb([
+        "title"
+      ]);
+      this.db = this.jio.__storage.__db;
+
+      this.setItemSpy = sinon.spy(localStorage, "setItem");
+      this.runSpy = sinon.spy(this.db, "run");
+    },
+    teardown: function () {
+      this.setItemSpy.restore();
+      delete this.setItemSpy;
+      this.runSpy.restore();
+      delete this.runSpy;
+    }
+  });
+
+  test("index document", function () {
+    var context = this,
+      doc = {
+        title: "document 1"
+      };
+    stop();
+
+    this.jio.post(doc)
+      .then(function () {
+        ok(
+          context.setItemSpy.calledOnce,
+          "setItem count " + context.setItemSpy.callCount
+        );
+        ok(
+          context.runSpy.calledOnce,
+          "run count " + context.runSpy.callCount
+        );
+        equal(
+          context.runSpy.firstCall.args[0],
+          "INSERT INTO jiosearch VALUES (:id, :title)",
+          "run first argument"
+        );
+        deepEqual(
+          context.runSpy.firstCall.args[1],
+          {
+            ":title": "document 1",
+            ":id": "1"
+          },
+          "run first argument"
+        );
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.put
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.put", {
+    setup: function () {
+      this.jio = jIO.createJIO({
+        type: "sql",
+        index_fields: [
+          "title"
+        ],
+        sub_storage: {
+          type: "sql200"
+        }
+      });
+      this.jio.__storage.__resetDb([
+        "title"
+      ]);
+      this.db = this.jio.__storage.__db;
+
+      this.setItemSpy = sinon.spy(localStorage, "setItem");
+      this.runSpy = sinon.spy(this.db, "run");
+    },
+    teardown: function () {
+      this.setItemSpy.restore();
+      delete this.setItemSpy;
+      this.runSpy.restore();
+      delete this.runSpy;
+    }
+  });
+
+  test("index document", function () {
+    var context = this,
+      doc = {
+        title: "document 1"
+      };
+    stop();
+
+    this.jio.put("1", doc)
+      .then(function () {
+        ok(
+          context.setItemSpy.calledOnce,
+          "setItem count " + context.setItemSpy.callCount
+        );
+        ok(
+          context.runSpy.calledOnce,
+          "run count " + context.runSpy.callCount
+        );
+        equal(
+          context.runSpy.firstCall.args[0],
+          "UPDATE jiosearch SET title = :title WHERE id=:id",
+          "run first argument"
+        );
+        deepEqual(
+          context.runSpy.firstCall.args[1],
+          {
+            ":title": "document 1",
+            ":id": "1"
+          },
+          "run first argument"
+        );
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.remove
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.remove", {
+    setup: function () {
+      this.jio = jIO.createJIO({
+        type: "sql",
+        index_fields: [
+          "title"
+        ],
+        sub_storage: {
+          type: "sql200"
+        }
+      });
+      this.jio.__storage.__resetDb([
+        "title"
+      ]);
+      this.db = this.jio.__storage.__db;
+
+      this.setItemSpy = sinon.spy(localStorage, "setItem");
+      this.runSpy = sinon.spy(this.db, "run");
+    },
+    teardown: function () {
+      this.setItemSpy.restore();
+      delete this.setItemSpy;
+      this.runSpy.restore();
+      delete this.runSpy;
+    }
+  });
+
+  test("remove index document", function () {
+    var context = this;
+    stop();
+
+    this.jio.remove("1")
+      .then(function () {
+        ok(
+          context.setItemSpy.calledOnce,
+          "setItem count " + context.setItemSpy.callCount
+        );
+        ok(
+          context.runSpy.calledOnce,
+          "run count " + context.runSpy.callCount
+        );
+        equal(
+          context.runSpy.firstCall.args[0],
+          "DELETE FROM jiosearch WHERE id=:id",
+          "run first argument"
+        );
+        deepEqual(
+          context.runSpy.firstCall.args[1],
+          {
+            ":id": "1"
+          },
+          "run first argument"
+        );
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // SqlStorage.buildQuery
+  /////////////////////////////////////////////////////////////////
+  module("SqlStorage.buildQuery", {
+    setup: function () {
+      this.jio = jIO.createJIO({
+        type: "sql",
+        sub_storage: {
+          type: "sql200"
+        }
+      });
+      this.jio.__storage.__resetDb([
+        "title"
+      ]);
+    }
+  });
+
+  test("list all documents", function () {
+    var context = this;
+    stop();
+
+    RSVP.all([
+      context.jio.post({
+        title: "document 1"
+      }),
+      context.jio.post({
+        title: "document 2"
+      })
+    ])
+      .then(function () {
+        return context.jio.allDocs();
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "1",
+              value: {
+                title: "document 1"
+              }
+            }, {
+              id: "2",
+              value: {
+                title: "document 2"
+              }
+            }],
+            total_rows: 2
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  test("search single field", function () {
+    var context = this;
+    stop();
+
+    RSVP.all([
+      context.jio.post({
+        title: "document 1"
+      }),
+      context.jio.post({
+        title: "image 2"
+      })
+    ])
+      .then(function () {
+        return context.jio.allDocs({
+          query: 'title: "doc"'
+        });
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "1",
+              value: {
+                title: "document 1"
+              }
+            }],
+            total_rows: 1
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  test("search multiple fields", function () {
+    this.jio.__storage.__resetDb([
+      "title",
+      "body"
+    ]);
+    var context = this;
+    stop();
+
+    RSVP.all([
+      context.jio.post({
+        title: "document 1",
+        body: "body document 1"
+      }),
+      context.jio.post({
+        title: "image 2",
+        body: "body document 2"
+      })
+    ])
+      .then(function () {
+        return context.jio.allDocs({
+          query: 'title: "doc" AND body: "doc"',
+          select_list: ['title']
+        });
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "1",
+              value: {
+                title: "document 1"
+              }
+            }],
+            total_rows: 1
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  test("limit results", function () {
+    this.jio.__storage.__resetDb([
+      "title"
+    ]);
+    var context = this,
+      documents = [],
+      i = 1;
+    stop();
+
+    while (i < 10) {
+      documents.push(context.jio.post({
+        title: "document " + i
+      }));
+      i = i + 1;
+    }
+
+    RSVP.all(documents)
+      .then(function () {
+        return context.jio.allDocs({
+          query: 'title: "doc"',
+          limit: [3, 2]
+        });
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "4",
+              value: {
+                title: "document 4"
+              }
+            }, {
+              id: "5",
+              value: {
+                title: "document 5"
+              }
+            }],
+            total_rows: 2
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+
+  test("sort results", function () {
+    this.jio.__storage.__resetDb([
+      "title",
+      "body"
+    ]);
+    var context = this;
+    stop();
+
+    RSVP.all([
+      context.jio.post({
+        title: "document 1",
+        body: "body document 1"
+      }),
+      context.jio.post({
+        title: "image 2",
+        body: "body document 2"
+      })
+    ])
+      .then(function () {
+        return context.jio.allDocs({
+          query: 'body: "doc"',
+          sort_on: [["title", "descending"]],
+          select_list: ["title"]
+        });
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "2",
+              value: {
+                title: "image 2"
+              }
+            }, {
+              id: "1",
+              value: {
+                title: "document 1"
+              }
+            }],
+            total_rows: 2
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+}(jIO, QUnit, sinon, localStorage));

test/tests.html

@@ -4,6 +4,7 @@
   <meta charset="utf-8" />
   <title>JIO Qunit/Sinon Unit Tests</title>
   <script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
+  <script src="../lib/sql.js/sql.js"></script>
   <script src="../dist/jio-latest.js"></script>
 
   <link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
@@ -56,6 +57,7 @@
   <script src="jio.storage/websqlstorage.tests.js"></script>
   <script src="jio.storage/fbstorage.tests.js"></script>
   <script src="jio.storage/httpstorage.tests.js"></script>
+  <script src="jio.storage/sqlstorage.tests.js"></script>
   <!--script src="../lib/jquery/jquery.min.js"></script>
   <script src="../src/jio.storage/xwikistorage.js"></script>
   <script src="jio.storage/xwikistorage.tests.js"></script-->