indexStorage: POST new API and qunit tests

src/jio.storage/indexstorage.js

 /*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
 /*global jIO: true, localStorage: true, setTimeout: true */
+/**
+ * JIO Index Storage.
+ * Manages indexes for specified storages.
+ * Description:
+ * {
+ *     "type": "index",
+ *     "indices": [
+ *        {"indexA",["field_A"]},
+ *        {"indexAB",["field_A","field_B"]}
+ *     ],
+ *     "storage": [
+ *         <sub storage description>,
+ *         ...
+ *     ]
+ * }
+ * Index file will contain
+ * {
+ *   "_id": "ipost_indices.json",
+ *   "indexA": {
+ *       "keyword_abc": ["some_id","some_other_id",...]
+ *   },
+ *   "indexAB": {
+ *       "keyword_abc": ["some_id"],
+ *       "keyword_def": ["some_id"]
+ *   }
+ * }
+ */
 jIO.addStorageType('indexed', function (spec, my) {
+
+  "use strict";
+  var that, priv = {}, spec;
+
   spec = spec || {};
-  var that = my.basicStorage(spec, my),
-    priv = {},
-    validatestate_sub_storage = spec.storage || false,
-    storage_object_name = 'jio/indexed_storage_object',
-    storage_file_object_name,
-    super_serialized = that.serialized;
+  that = my.basicStorage(spec, my);
 
-  priv.sub_storage_spec = spec.storage || {
-    type: 'base'
-  };
-  priv.sub_storage_string = JSON.stringify(priv.sub_storage_spec);
+  priv.indices = spec.indices;
+  priv.substorage_key = "sub_storage";
+  priv.substorage = spec[priv.substorage_key];
+  priv.index_indicator = spec.sub_storage.application_name || "index";
+  priv.index_suffix = priv.index_indicator + "_indices.json";
 
-  storage_file_object_name = 'jio/indexed_file_object/' +
-    priv.sub_storage_string;
+  my.env = my.env || spec.env || {};
 
-  that.serialized = function () {
-    var o = super_serialized();
-    o.storage = priv.sub_storage_spec;
+  that.specToStore = function () {
+    var o = {};
+    o[priv.substorage_key] = priv.substorage;
+    o.env = my.env;
     return o;
   };
 
-  that.validateState = function () {
-    if (!validatestate_sub_storage) {
-      return 'Need at least one parameter: "storage" ' +
-        'containing storage specifications.';
+  /**
+   * Generate a new uuid
+   * @method generateUuid
+   * @return {string} The new uuid
+   */
+  priv.generateUuid = function () {
+    var S4 = function () {
+      var i, string = Math.floor(
+        Math.random() * 0x10000 /* 65536 */
+      ).toString(16);
+      for (i = string.length; i < 4; i += 1) {
+        string = "0" + string;
+      }
+      return string;
+    };
+    return S4() + S4() + "-" +
+      S4() + "-" +
+      S4() + "-" +
+      S4() + "-" +
+      S4() + S4() + S4();
+  };
+
+  /**
+  * Escape string before storing
+  * @method sanitizeValue
+  * @param  {string} s The string to be sanitized
+  * @return {string} The sanitized string
+  */
+  priv.sanitizeValue = function (s) {
+    return s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')
+  };
+
+  /**
+  * Get number of elements in object
+  * @method getObjectSize
+  * @param  {object} obj The object to check
+  * @return {number} size The amount of elements in the object
+  */
+  priv.getObjectSize = function (obj) {
+    var size = 0, key;
+    for (key in obj) {
+        if (obj.hasOwnProperty(key)) {
+          size += 1;
+        }
     }
-    return '';
+    return size;
   };
 
-  priv.secureDocId = function (string) {
-    var split = string.split('/'),
-      i;
-    if (split[0] === '') {
-      split = split.slice(1);
+  /**
+   * Creates an empty indices array
+   * @method createEmptyIndexArray
+   * @param  {array} indices An array of indices (optional)
+   * @return {object} The new index array
+   */
+  priv.createEmptyIndexArray = function (indices) {
+    var obj, i, j = priv.indices.length,
+      new_index_object = {}, new_index_name;
+
+    if (indices === undefined) {
+      for (i = 0; i < j; i += 1) {
+        new_index_name = priv.indices[i]["name"];
+        new_index_object[new_index_name] = {};
+      }
     }
-    for (i = 0; i < split.length; i += 1) {
-      if (split[i] === '') {
-        return '';
+    return new_index_object;
+  };
+
+  /**
+   * Find id in indices
+   * @method docidInIndex
+   * @param  {object} indices The file containing the indeces
+   * @param  {object} doc The document which should be added to the index
+   * @return {boolean} true/false
+   */
+  priv.docidInIndex = function (indices, doc) {
+    var i, j, l = priv.indices.length, elements_to_check,
+      index, index_name, index_length;
+
+    // loop indices
+    for (i = 0; i < l; i += 1) {
+      index = priv.indices[i];
+      index_name = index["name"];
+      index_length = index.fields.length;
+      elements_to_check = priv.getObjectSize(indices[index_name]);
+
+      if (elements_to_check > 0) {
+        for (var key in indices[index_name]) {
+          var obj = indices[index_name][key];
+          for (var prop in obj) {
+            if (obj[prop] === doc._id) {
+              return true;
+            }
+          }
+        }
       }
     }
-    return split.join('%2F');
+    return false;
+  }
+  /**
+   * Adds entries to indices
+   * @method createEmptyIndexArray
+   * @param  {object} indices The file containing the indeces
+   * @param  {object} doc The document which should be added to the index
+   */
+  priv.updateIndeces = function (indices, doc) {
+    var i, j, k, m, value,
+      index, index_name, index_length, index_field_array,
+      l = priv.indices.length,
+      docid = doc._id;
+
+    // loop indices
+    for (i = 0; i < l; i += 1) {
+      index = priv.indices[i];
+      index_name = index["name"];
+      index_length = index.fields.length;
+      index_field_array = [];
+
+      // loop index fields [keywords]
+      for (j = 0; j < index_length; j += 1) {
+        value = doc[index.fields[j]];
+        if (value !== undefined) {
+          index_field_array.push(value);
+        }
+      }
+
+      m = index_field_array.length;
+      if (m) {
+        for (k = 0; k < m; k += 1) {
+          if (indices[index_name] !== undefined) {
+            if (indices[index_name][index_field_array[k]] === undefined) {
+              indices[index_name][index_field_array[k]] = [];
+            }
+            indices[index_name][index_field_array[k]].push(docid);
+          }
+        }
+      }
+    }
+    return indices;
   };
 
-  priv.indexStorage = function () {
-    var obj = localStorage.getItem(storage_object_name) || {};
-    obj[priv.sub_storage_spec] = new Date().getTime();
-    localStorage.setItem(storage_object_name, obj);
+  /**
+   * Post document to substorage and create/update index file(s)
+   * @method post
+   * @param  {object} command The JIO command
+   */
+  that.post = function (command) {
+    var f = {}, indices, doc, docid;
+    doc = command.cloneDoc();
+    docid = command.getDocId();
+    if (typeof docid !== "string") {
+      doc._id = priv.generateUuid();
+      docid = doc._id;
+    }
+    f.getIndices = function () {
+      var option = command.cloneOption();
+      if (option.max_retry === 0) {
+        option.max_retry = 3;
+      }
+      that.addJob(
+        "get",
+        priv.substorage,
+        priv.index_suffix,
+        option,
+        function (response) {
+          indices = response;
+          f.postDocument("put");
+        },
+        function (err) {
+          switch (err.status) {
+          case 404:
+            indices = priv.createEmptyIndexArray();
+            f.postDocument("post");
+            break;
+          default:
+            err.message = "Cannot retrieve index array";
+            that.error(err);
+            break;
+          }
+        }
+      );
+    };
+    f.postDocument = function (index_update_method) {
+      // if the index file already has an entry with this id,
+      // the document already exists
+      if (priv.docidInIndex(indices, doc)) {
+        // POST the document already exists
+        that.error({
+          "status": 409,
+          "statusText": "Conflicts",
+          "error": "conflicts",
+          "message": "Cannot create a new document",
+          "reason": "Document already exists"
+        });
+        return;
+      } else {
+        indices = priv.updateIndeces(indices, doc);
+        that.addJob(
+          "post",
+          priv.substorage,
+          doc,
+          command.cloneOption(),
+          function () {
+            f.sendIndices(index_update_method);
+          },
+          function (err) {
+            switch (err.status) {
+            case 409:
+              // file already exists
+              f.sendIndices(index_update_method);
+              break;
+            default:
+              err.message = "Cannot upload document";
+              that.error(err);
+              break;
+            }
+          }
+        );
+      }
+    };
+    f.sendIndices = function (method) {
+      indices._id = priv.index_suffix;
+      that.addJob(
+        method,
+        priv.substorage,
+        indices,
+        command.cloneOption(),
+        function () {
+          that.success({
+            "ok": true,
+            "id": docid
+          });
+        },
+        function (err) {
+          // xxx do we try to delete the posted document ?
+          err.message = "Cannot save index file";
+          that.error(err);
+        }
+      );
+    };
+    f.getIndices();
   };
 
+/*
+  /**
+   * @method formatToFileObject
+   * @param  {} row
+   * @return {} obj
+  *//*
   priv.formatToFileObject = function (row) {
     var k, obj = {
       _id: row.id
@@ -62,7 +309,12 @@ jIO.addStorageType('indexed', function (spec, my) {
     }
     return obj;
   };
-
+*/
+  /**
+   * @method allDocs
+   * @param  {} files_object
+   * @return {} obj
+  *//*
   priv.allDocs = function (files_object) {
     var k, obj = {
       rows: []
@@ -79,7 +331,11 @@ jIO.addStorageType('indexed', function (spec, my) {
     obj.total_rows = obj.rows.length;
     return obj;
   };
-
+*/
+  /**
+   * @method setFileArray
+   * @param  {} file_array
+  *//*
   priv.setFileArray = function (file_array) {
     var i, obj = {};
     for (i = 0; i < file_array.length; i += 1) {
@@ -87,67 +343,84 @@ jIO.addStorageType('indexed', function (spec, my) {
     }
     localStorage.setItem(storage_file_object_name, obj);
   };
-
+*/
+  /**
+   * @method getFileObject
+   * @param  {} docid
+   * @return {} obj
+  *//*
   priv.getFileObject = function (docid) {
     var obj = localStorage.getItem(storage_file_object_name) || {};
     return obj[docid];
   };
-
+*/
+  /**
+   * @method addFile
+   * @param  {} file_obj
+  *//*
   priv.addFile = function (file_obj) {
     var obj = localStorage.getItem(storage_file_object_name) || {};
     obj[file_obj._id] = file_obj;
     localStorage.setItem(storage_file_object_name, obj);
   };
-
+*/
+  /**
+   * @method removeFile
+   * @param  {} docid
+  *//*
   priv.removeFile = function (docid) {
     var obj = localStorage.getItem(storage_file_object_name) || {};
     delete obj[docid];
     localStorage.setItem(storage_file_object_name, obj);
   };
-
+*/
   /**
    * updates the storage.
    * It will retreive all files from a storage. It is an asynchronous task
    * so the update can be on going even if IndexedStorage has already
    * returned the result.
    * @method update
-   */
+  *//*
   priv.update = function () {
-    var success = function (val) {
-      priv.setFileArray(val.rows);
-    };
-    that.addJob('allDocs', priv.sub_storage_spec, null, {
-      max_retry: 3
-    }, success, function () {});
-  };
-
-  that.post = function (command) {
-    that.put(command);
+    that.addJob(
+      'allDocs',
+      priv.sub_storage_spec,
+      null,
+      {max_retry: 3},
+      function (response) {
+        priv.setFileArray(response.rows);
+      },
+      function () {}
+    );
   };
-
+*/
   /**
-   * Saves a document.
+   * Add put job to substorage and create/update index file(s)
    * @method put
-   */
+   * @param  {object} command The JIO command
+   *//*
   that.put = function (command) {
     var cloned_doc = command.cloneDoc(),
-      cloned_option = command.cloneOption(),
-      success = function (val) {
+      cloned_option = command.cloneOption();
+    // create/update indexStorage
+
+    //  fwd job
+    that.addJob('put', priv.sub_storage_spec, cloned_doc,
+      cloned_option,
+      function (response) {
         priv.update();
-        that.success(val);
+        that.success(response);
       },
-      error = function (err) {
-        that.error(err);
-      };
-    priv.indexStorage();
-    that.addJob('put', priv.sub_storage_spec, cloned_doc,
-      cloned_option, success, error);
-  }; // end put
+      function (error) {
+        that.error(error);
+      }
+    );
+  };
 
-  /**
+  *//**
    * Loads a document.
    * @method get
-   */
+  *//*
   that.get = function (command) {
     // jslint unused var file_array
     var success = function (val) {
@@ -177,11 +450,27 @@ jIO.addStorageType('indexed', function (spec, my) {
       get();
     }
   }; // end get
-
-  /**
+ *//**
+   * Removes a document.
+   * @method remove
+   *//*
+  that.remove = function (command) {
+    var success = function (val) {
+      priv.removeFile(command.getDocId());
+      priv.update();
+      that.success(val);
+    },
+      error = function (err) {
+        that.error(err);
+      };
+    that.addJob('remove', priv.sub_storage_spec, command.cloneDoc(),
+      command.cloneOption(), success, error);
+  };
+  *//**
    * Gets a document list.
    * @method allDocs
    */
+  /*
   that.allDocs = function (command) {
     var obj = localStorage.getItem(storage_file_object_name),
       success,
@@ -204,23 +493,6 @@ jIO.addStorageType('indexed', function (spec, my) {
         command.cloneOption(), success, error);
     }
   }; // end allDocs
-
-  /**
-   * Removes a document.
-   * @method remove
-   */
-  that.remove = function (command) {
-    var success = function (val) {
-      priv.removeFile(command.getDocId());
-      priv.update();
-      that.success(val);
-    },
-      error = function (err) {
-        that.error(err);
-      };
-    that.addJob('remove', priv.sub_storage_spec, command.cloneDoc(),
-      command.cloneOption(), success, error);
-  }; // end remove
-
+  */
   return that;
 });

test/jiotests.js

@@ -2619,9 +2619,64 @@ test ('Remove document', function () {
     o.mytest('DummyStorageAllOK,3tries: remove document.',{ok:true,id:'file'});
     o.jio.stop();
 });
+*/
+module ("Jio IndexStorage");
+
+test ("Post", function () {
+
+    var o = generateTools(this);
+
+    o.jio = JIO.newJio({
+        "type": "indexed",
+        "indices": [{
+          "name":"indexA",
+          "fields":["findMeA"]
+        },{
+          "name":"indexAB",
+          "fields":["findMeA","findMeB"]
+        }],
+        "sub_storage": {
+          "type": "local",
+          "username": "ipost",
+          "application_name": "ipost"
+        }
+    });
+
+    // post without id
+    o.spy (o, "status", undefined, "Post without id");
+    o.jio.post({}, o.f);
+    o.tick(o);
+
+    // post non empty document
+    o.doc = {"_id": "some_id", "title": "myPost1",
+      "findMeA":"keyword_abc", "findMeB":"keyword_def"
+    };
+    o.spy (o, "value", {"ok": true, "id": "some_id"}, "Post document");
+    o.jio.post(o.doc, o.f);
+    o.tick(o);
 
-module ('Jio IndexedStorage');
+    // check document
+    o.indexPost = {
+      "indexAB": {"keyword_abc":["some_id"], "keyword_def":["some_id"]},
+      "indexA": {"keyword_abc":["some_id"]}
+    };
+    deepEqual(
+        o.jio.get("ipost_indices.json"),
+        o.indexPost,
+        "Check index file"
+    );
 
+    // post and document already exists
+    o.doc = {"_id": "some_id", "title": "myPost2",
+      "findMeA":"keyword_ghi", "findMeB":"keyword_jkl"
+    }
+    o.spy (o, "status", 409, "Post and document already exists");
+    o.jio.post(o.doc, o.f);
+    o.tick(o);
+
+    o.jio.stop();
+});
+/*
 test ('Document load', function () {
     var o = {}; o.clock = this.sandbox.useFakeTimers();
     o.clock.tick(base_tick);
@@ -2665,27 +2720,6 @@ test ('Document load', function () {
     o.jio.stop();
 });
 
-test ('Document save', function () {
-    var o = {}; o.clock = this.sandbox.useFakeTimers();
-    o.clock.tick(base_tick);
-    o.jio = JIO.newJio({type:'indexed',
-                        storage:{type:'dummyall3tries',
-                                 username:'indexsave'}});
-    o.f = function (err,val) {
-        if (err) {
-            err = err.status;
-        }
-        deepEqual (err || val,{ok:true,id:'file'},'document save');
-    };
-    this.spy(o,'f');
-    o.jio.put({_id:'file',content:'content'},{max_retry:3},o.f);
-    o.clock.tick(2000);
-    if (!o.f.calledOnce){
-        ok (false, 'no response / too much results');
-    }
-    o.jio.stop();
-});
-
 test ('Get document list', function () {
     var o = {}; o.clock = this.sandbox.useFakeTimers();
     o.clock.tick(base_tick);
@@ -2742,7 +2776,8 @@ test ('Remove document', function () {
 
     o.jio.stop();
 });
-
+*/
+/*
 module ('Jio CryptedStorage');
 
 test ('Document save' , function () {

test/jiotests_withoutrequirejs.html

@@ -28,9 +28,11 @@
           src="../src/jio.storage/replicaterevisionstorage.js">
   </script>
 
-  <!-- webDav -->
   <script type="text/javascript" src="../lib/jquery/jquery.min.js"></script>
   <script type="text/javascript" src="../src/jio.storage/davstorage.js">
+  </script>
+  <script type="text/javascript" src="../src/jio.storage/indexstorage.js">
+  </script>
 
   </script>
   <script type="text/javascript" src="./jiotests.js"></script>