indexStorage: fixed ALLDOCS complex queries, added QUnit tests

src/jio.storage/indexstorage.js

@@ -10,6 +10,10 @@
  *        {"indexA",["field_A"]},
  *        {"indexAB",["field_A","field_B"]}
  *     ],
+ *     "field_types": {
+ *        "field_A": "dateTime",
+ *        "field_B": "string"
+ *      },
  *     "storage": [
  *         <sub storage description>,
  *         ...
@@ -17,7 +21,7 @@
  * }
  * Index file will contain
  * {
- *   "_id": "ipost_indices.json",
+ *   "_id": "app-name_indices.json",
  *   "indexA":
  *      "fieldA": {
  *        "keyword_abc": ["some_id","some_other_id",...]
@@ -54,6 +58,7 @@ jIO.addStorageType('indexed', function (spec, my) {
   that = my.basicStorage(spec, my);
 
   priv.indices = spec.indices;
+  priv.field_types = spec.field_types;
   priv.substorage_key = "sub_storage";
   priv.substorage = spec[priv.substorage_key];
   priv.index_indicator = spec.sub_storage.application_name || "index";
@@ -164,8 +169,8 @@ jIO.addStorageType('indexed', function (spec, my) {
    * @returns {number} i Position of element in array
    */
   priv.getPositionInArray = function (element, array) {
-    var i;
-    for (i = 0; i < array.length; i += 1) {
+    var i, l = array.length;
+    for (i = 0; i < l; i += 1) {
       if (array[i] === element) {
         return i;
       }
@@ -291,36 +296,45 @@ jIO.addStorageType('indexed', function (spec, my) {
   };
 
   /**
-   * Check available indices to find the best one. This index must have
-   * all "id" parameters of the query and if it also contains all values from
-   * the select-list, it can be used to run the whole query plus return results
+   * Check available indices to find the best one.
+   * TODOS: NOT NICE, redo
    * @method findBestIndexForQuery
-   * @param  {object} indices The index file
+   * @param  {object} syntax of query
    * @returns {object} response The query object constructed from Index file
    */
-  priv.findBestIndexForQuery = function (indices, syntax) {
-    var i, j, k, l, m, n, o, p,
-      search_ids = [],
-      select_ids = syntax.filter.select_list,
-      index, query_param, search_param, use_index = [];
-
-    // array of necessary query ids
-    if (syntax.query.query_list === undefined) {
-      search_ids.push(syntax.query.id);
-    } else {
-      for (j = 0; j < syntax.query.query_list.length; j += 1) {
-        search_ids.push(syntax.query.query_list[j].id);
-      }
-    }
+  priv.findBestIndexForQuery = function (syntax) {
+    var i, j, k, l, n, p, o, element, key, block,
+      search_ids, use_index = [], select_ids = {}, index, query_param,
+      // need to parse into object
+      current_query = jIO.ComplexQueries.parse(syntax.query);
 
     // loop indices
     for (i = 0; i < priv.indices.length; i += 1) {
+      search_ids = [];
+      block = false;
       index = {};
       index.reference = priv.indices[i];
       index.reference_size = index.reference.fields.length;
-      o = search_ids.length;
-      for (k = 0; k < o; k += 1) {
-        // always check the first element in the array because it's spliced
+
+      // rebuild search_ids for iteration
+      if (current_query.query_list === undefined) {
+        search_ids.push(current_query.id);
+      } else {
+        for (j = 0; j < current_query.query_list.length; j += 1) {
+          if (priv.getPositionInArray(current_query.query_list[j].id,
+              search_ids) === null) {
+            search_ids.push(current_query.query_list[j].id);
+          }
+        }
+      }
+      // rebuild select_ids
+      for (o = 0; o < syntax.filter.select_list.length; o += 1) {
+        element = syntax.filter.select_list[o];
+        select_ids[element] = true;
+      }
+
+      // loop search ids and find matches in index
+      for (k = 0; k < search_ids.length; k += 1) {
         query_param = search_ids[0];
         for (l = 0; l < index.reference_size; l += 1) {
           if (query_param === index.reference.fields[l]) {
@@ -332,41 +346,35 @@ jIO.addStorageType('indexed', function (spec, my) {
         }
       }
 
-      // empty search_ids = index can be used, now check results
+      // search_ids empty  = all needed search fields found on index
       if (search_ids.length === 0) {
-        // can't return results if empty select_list (all fields)
-        if (select_ids.length === 0) {
+        p = priv.getObjectSize(select_ids);
+        if (p === 0) {
           use_index.push({
             "name": index.reference.name,
             "search": true,
             "results": false
           });
         } else {
-          p = select_ids.length;
-          for (m = 0; m < p; m += 1) {
-            search_param = select_ids[0];
-            for (n = 0; n < index.reference_size; n += 1) {
-              if (search_param === index.reference.fields[n]) {
-                select_ids.splice(
-                  priv.getPositionInArray(search_param, select_ids),
-                  1
-                );
-              }
+          for (n = 0; n < index.reference_size; n += 1) {
+            delete select_ids[index.reference.fields[n]];
+          }
+          for (key in select_ids) {
+            if (select_ids.hasOwnProperty(key)) {
+              use_index.push({
+                "name": index.reference.name,
+                "search": true,
+                "results": false
+              });
+              block = true;
             }
           }
-          // empty select_list = index can do do query and results!
-          if (select_ids.length === 0) {
+          if (block === false) {
             use_index.push({
               "name": index.reference.name,
               "search": true,
               "results": true
             });
-          } else {
-            use_index.push({
-              "name": index.reference.name,
-              "search": true,
-              "results": false
-            });
           }
         }
       }
@@ -376,13 +384,78 @@ jIO.addStorageType('indexed', function (spec, my) {
 
   /**
    * Converts the indices file into an object usable by complex queries
-   * @method convertIndicesToQueryObject
+   * @method constructQueryObject
    * @param  {object} indices The index file
    * @returns {object} response The query object constructed from Index file
    */
-  priv.convertIndicesToQueryObject = function (indices, query_syntax) {
-    var use_index = priv.findBestIndexForQuery(indices, query_syntax);
-    return indices;
+  priv.constructQueryObject = function (indices, query_syntax) {
+    var j, k, l, m, n, use_index, index,
+      index_name, field_names, field, key, element,
+      query_index, query_object = [], field_name,
+      entry;
+
+    // returns index-to-use|can-do-query|can-do-query-and-results
+    use_index = priv.findBestIndexForQuery(query_syntax);
+
+    if (use_index.length > 0) {
+      for (j = 0; j < use_index.length; j += 1) {
+        index = use_index[j];
+
+        // NOTED: the index could be used to:
+        // (a) get all document ids matching query
+        // (b) get all document ids and results (= run complex query on index)
+        // right now, only (b) is supported, because the complex query is
+        // a single step process. If it was possible to first get the 
+        // relevant document ids, then get the results, the index could be
+        // used to do the first step plus use GET on the returned documents
+        if (index.search && index.results) {
+          index_name = use_index[j].name;
+          query_index = indices[index_name];
+
+          // get fieldnames from this index
+          for (k = 0; k < priv.indices.length; k += 1) {
+            if (priv.indices[k].name === use_index[j].name) {
+              field_names = priv.indices[k].fields;
+            }
+          }
+          for (l = 0; l < field_names.length; l += 1) {
+            field_name = field_names[l];
+            // loop entries for this field name
+            field = query_index[field_name];
+            for (key in field) {
+              if (field.hasOwnProperty(key)) {
+                element = field[key];
+                // key can be "string" or "number" right now
+                if (priv.field_types[field_name] === "number") {
+                  key = +key;
+                }
+                for (m = 0; m < element.length; m += 1) {
+                  if (priv.searchIndexByValue(
+                      query_object,
+                      element[m],
+                      "bool"
+                    )) {
+                    // loop object
+                    for (n = 0; n < query_object.length; n += 1) {
+                      entry = query_object[n];
+                      if (entry.id === element[m]) {
+                        entry[field_name] = key;
+                      }
+                    }
+                  } else {
+                    entry = {};
+                    entry.id = element[m];
+                    entry[field_name] = key;
+                    query_object.push(entry);
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    return query_object;
   };
   /**
    * Build the alldocs response from the index file (overriding substorage)
@@ -490,7 +563,7 @@ jIO.addStorageType('indexed', function (spec, my) {
    * @param  {object} command The JIO command
    * @param  {string} source The source of the function call
    */
-  priv.postOrput = function (command, source) {
+  priv.postOrPut = function (command, source) {
     var f = {}, indices, doc, docid;
     doc = command.cloneDoc();
     docid = command.getDocId();
@@ -618,7 +691,7 @@ jIO.addStorageType('indexed', function (spec, my) {
    * @param  {object} command The JIO command
    */
   that.post = function (command) {
-    priv.postOrput(command, 'POST');
+    priv.postOrPut(command, 'POST');
   };
 
   /**
@@ -627,7 +700,7 @@ jIO.addStorageType('indexed', function (spec, my) {
    * @param  {object} command The JIO command
    */
   that.put = function (command) {
-    priv.postOrput(command, 'PUT');
+    priv.postOrPut(command, 'PUT');
   };
 
   /**
@@ -636,7 +709,7 @@ jIO.addStorageType('indexed', function (spec, my) {
    * @param  {object} command The JIO command
    */
   that.putAttachment = function (command) {
-    priv.postOrput(command, 'PUTATTACHMENT');
+    priv.postOrPut(command, 'PUTATTACHMENT');
   };
 
   /**
@@ -782,7 +855,8 @@ jIO.addStorageType('indexed', function (spec, my) {
   // ]
   //}
   that.allDocs = function (command) {
-    var f = {}, option, all_docs_response, query_object, query_syntax;
+    var f = {}, option, all_docs_response, query_object, query_syntax,
+      query_response;
     option = command.cloneOption();
     if (option.max_retry === 0) {
       option.max_retry = 3;
@@ -795,14 +869,61 @@ jIO.addStorageType('indexed', function (spec, my) {
         priv.index_suffix,
         option,
         function (response) {
-          // should include_docs be possible besides complex queries?
           query_syntax = command.getOption('query');
+
           if (query_syntax !== undefined) {
-            // check to see if index can do the job
-            query_object = priv.convertIndicesToQueryObject(
-              response,
-              query_syntax
-            );
+
+            // build complex query object
+            query_object = priv.constructQueryObject(response, query_syntax);
+
+            if (query_object.length === 0) {
+              that.addJob(
+                "allDocs",
+                priv.substorage,
+                undefined,
+                option,
+                function (data) {
+                  that.success(data);
+                },
+                function (err) {
+                  switch (err.status) {
+                  case 405:
+                    that.error({
+                      "status": 405,
+                      "statusText": "Method Not Allowed",
+                      "error": "method_not_allowed",
+                      "message": "Method not allowed",
+                      "reason": "Could not run AllDocs on this storage"
+                    });
+                    break;
+                  default:
+                    that.error({
+                      "status": 404,
+                      "statusText": "Not Found",
+                      "error": "not_found",
+                      "message": "Could not run allDocs command",
+                      "reason": "There are no documents in the storage"
+                    });
+                    break;
+                  }
+                }
+              );
+            } else {
+              // we can use index, run query on index
+              query_response =
+                jIO.ComplexQueries.query({
+                  query: query_syntax.query,
+                  filter: {
+                    sort_on: query_syntax.filter.sort_on,
+                    limit: query_syntax.filter.limit,
+                    select_list: query_syntax.filter.select_list
+                  },
+                  wildcard_character: query_syntax.wildcard_character
+                },
+                  query_object
+                  );
+              that.success(query_response);
+            }
           } else if (command.getOption('include_docs')) {
             priv.allDocsResponseFromIndex(response, true, option);
           } else {

test/jiotests.js

@@ -244,7 +244,15 @@ generateTools = function (sinon) {
       o.server.respondWith(method, url,
         [status, { "Content-Type": 'application/xml' }, response]
       );
-    }
+    };
+    o.sortArrayById = function(field, reverse, primer){
+      var key = function (x) {return primer ? primer(x[field]) : x[field]};
+
+      return function (a,b) {
+        var A = key(a), B = key(b);
+        return ( (A < B) ? -1 : ((A > B) ? 1 : 0) ) * [-1,1][+!!reverse];
+      }
+    };
 
     return o;
 },
@@ -1097,7 +1105,7 @@ test ("AllDocs", function(){
       "Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
       "Milos Forman", "Christopher Nolan", " Martin Scorsese"
     ]
-    // set documents
+
     for (i = 0; i < m; i += 1) {
       o.fakeDoc = {};
       o.fakeDoc._id = "doc_"+i;
@@ -2790,6 +2798,10 @@ test ("Post", function () {
             {"name":"indexA", "fields":["findMeA"]},
             {"name":"indexAB", "fields":["findMeA","findMeB"]}
         ],
+        "field_types": {
+          "findMeA": "string",
+          "findMeB": "string"
+        },
         "sub_storage": {
           "type": "local",
           "username": "ipost",
@@ -2863,6 +2875,10 @@ test ("Put", function(){
               {"name":"indexA", "fields":["author"]},
               {"name":"indexAB", "fields":["author","year"]}
           ],
+          "field_types": {
+            "author": "string",
+            "year": "number"
+          },
           "sub_storage": {
             "type": "local",
             "username": "iput",
@@ -3035,6 +3051,10 @@ test ("PutAttachment", function(){
               {"name":"indexA", "fields":["author"]},
               {"name":"indexAB", "fields":["author","year"]}
           ],
+          "field_types": {
+            "author": "string",
+            "year": "number"
+          },
           "sub_storage": {
             "type": "local",
             "username": "iputatt",
@@ -3142,6 +3162,10 @@ test ("Get", function(){
               {"name":"indexA", "fields":["author"]},
               {"name":"indexAB", "fields":["author","year"]}
           ],
+          "field_types": {
+            "author": "string",
+            "year": "number"
+          },
           "sub_storage": {
             "type": "local",
             "username": "iget",
@@ -3208,6 +3232,10 @@ test ("Remove", function(){
           {"name":"indexA", "fields":["author"]},
           {"name":"indexAB", "fields":["author","year"]}
       ],
+      "field_types": {
+            "author": "string",
+            "year": "number"
+      },
       "sub_storage": {
         "type": "local",
         "username": "irem",
@@ -3372,6 +3400,10 @@ test ("AllDocs", function () {
           {"name":"indexA", "fields":["author"]},
           {"name":"indexAB", "fields":["author","year"]}
       ],
+      "field_types": {
+        "author": "string",
+        "year": "number"
+      },
       "sub_storage": {
         "type": "local",
         "username": "iall",
@@ -3461,25 +3493,158 @@ test ("AllDocs", function () {
   o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
   o.jio.allDocs({"include_docs":true}, o.f);
   o.tick(o);
-  /*
-  // complex queries
-  o.thisShouldBeTheAnswer3 = {"nothing here":"yet"}
-  o.spy(o, "value", o.thisShouldBeTheAnswer3,
-    "allDocs (complex queries year >= 1985)");
-  o.jio.allDocs({
-    "query":{
-      "query":jIO.ComplexQueries.parse('(year: >= "1985" AND author:"D%")'),
-      "filter": {
-          "limit":[0,2],
-          "sort_on":[['key','descending']],
-          "select_list":['author','year']
+
+  o.jio.stop();
+});
+
+test ("AllDocs Complex Queries", function () {
+
+  var o = generateTools(this), i, m = 15;
+
+    o.jio = JIO.newJio({
+      "type": "indexed",
+      "indices": [
+          {"name":"indexA", "fields":["director"]},
+          {"name":"indexAB", "fields":["title","year"]}
+          //,
+          //{"name":"indexABC", "fields":["title","year","director"]}
+      ],
+      "field_types": {
+        "director": "string",
+        "title": "string",
+        "year": "number"
       },
-      "wildcard_character":'%'
+      "sub_storage": {
+        "type": "local",
+        "username": "icomplex",
+        "application_name": "acomplex"
+      }
+    });
+    o.localpath = "jio/localstorage/icomplex/acomplex";
+
+    // sample data
+    o.titles = ["Shawshank Redemption", "Godfather", "Godfather 2",
+      "Pulp Fiction", "The Good, The Bad and The Ugly", "12 Angry Men",
+      "The Dark Knight", "Schindlers List",
+      "Lord of the Rings - Return of the King", "Fight Club",
+      "Star Wars Episode V", "Lord Of the Rings - Fellowship of the Ring",
+      "One flew over the Cuckoo's Nest", "Inception", "Godfellas"
+    ];
+    o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
+      1975,2010,1990
+    ];
+    o.director = ["Frank Darabont", "Francis Ford Coppola",
+      "Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
+      "Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
+      "Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
+      "Milos Forman", "Christopher Nolan", " Martin Scorsese"
+    ]
+
+    for (i = 0; i < m; i += 1) {
+      o.fakeDoc = {};
+      o.fakeDoc._id = ""+i;
+      o.fakeDoc.title = o.titles[i];
+      o.fakeDoc.year = o.years[i];
+      o.fakeDoc.director = o.director[i];
+      o.jio.put(o.fakeDoc);
+      o.clock.tick(1000);
     }
-  }, o.f);
-  o.tick(o);
-  */
-  o.jio.stop();
+
+    // response
+    o.allDocsResponse = {};
+    o.allDocsResponse.rows = [];
+    o.allDocsResponse.total_rows = 15;
+    for (i = 0; i < m; i += 1) {
+      o.allDocsResponse.rows.push({
+        "id": ""+i,
+        "key": ""+i,
+        "value": {}
+      });
+    };
+
+    // alldocs
+    o.jio.allDocs(function (e, r) {
+      var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
+      deepEqual(
+        {"total_rows":r.total_rows,"rows":x}, o.allDocsResponse,
+          "AllDocs response generated from index"
+      );
+    });
+    o.clock.tick(1000);
+
+    // include docs
+    o.allDocsResponse2 = {};
+    o.allDocsResponse2.rows = [];
+    o.allDocsResponse2.total_rows = 15;
+    for (i = 0; i < m; i += 1) {
+      o.allDocsResponse2.rows.push({
+        "id": ""+i,
+        "key": ""+i,
+        "value": {},
+        "doc": localstorage.getItem(o.localpath+"/"+i)
+      });
+    };
+
+    // alldocs
+    o.jio.allDocs({"include_docs":true}, function(e,r) {
+      var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
+      deepEqual(
+        {"total_rows":r.total_rows,"rows":x}, o.allDocsResponse2,
+          "AllDocs response generated from index (include docs)"
+      );
+    });
+    o.clock.tick(1000);
+
+    // complex queries
+    o.thisShouldBeTheAnswer4 = [
+        {"title": "Inceptions", "year": 2010},
+        {"title": "The Dark Knight", "year": 2008},
+        {"title": "Lord of the Rings - Return of the King", "year": 2003},
+        {"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001},
+        {"title": "Fight Club", "year": 1999}
+    ];
+    o.spy(o, "value", o.thisShouldBeTheAnswer4,
+      "allDocs (complex queries year >= 1980, index used to do query)");
+    o.jio.allDocs({
+      "query":{
+        // "query":'(year: >= "1980" AND year: < "2000")',
+        "query":'(year: >= "1980")',
+        "filter": {
+            "limit":[0,5],
+            "sort_on":[['year','descending']],
+            "select_list":['title','year']
+        },
+        "wildcard_character":'%'
+      }
+    }, o.f);
+    o.tick(o);
+
+    // complex queries
+    o.thisShouldBeTheAnswer5 = [
+        {"director": "Christopher Nolan", "year": 2010},
+        {"director": "Christopher Nolan", "year": 2008},
+        {"director": "Peter Jackson", "year": 2003},
+        {"director": "Peter Jackson", "year": 2001},
+        {"director": "David Fincher", "year": 1999}
+    ];
+
+    o.spy(o, "value", o.thisShouldBeTheAnswer5,
+      "allDocs (complex queries year >= 1980, can't use index)");
+    o.jio.allDocs({
+      "query":{
+        // "query":'(year: >= "1980" AND year: < "2000")',
+        "query":'(year: >= "1980")',
+        "filter": {
+            "limit":[0,5],
+            "sort_on":[['year','descending']],
+            "select_list":['director','year']
+        },
+        "wildcard_character":'%'
+      }
+    }, o.f);
+    o.tick(o);
+
+    o.jio.stop();
 });
 /*
 module ('Jio CryptedStorage');