Merge branch 'master' of https://lab.nexedi.com/nexedi/jio into HEAD

Conflicts:
	Gruntfile.js
	dist/jio-latest.js
	dist/jio-latest.min.js

README.md

-## Javascript Input/Output
+# jIO
 
-**jIO is a client-side JavaScript library to manage documents across multiple
-  storages.**
+jIO is a promised-based JavaScript library that offers connectors to many storages (Dropbox, webdav, IndexedDB, GDrive, ...) using a single API. jIO supports offline use, replication, encryption and synchronization as well as querying of stored documents and attachments allowing to build complex client-side centric web applications.
 
-### Getting Started
+### jIO Documentation
 
-To set up jIO you should include jio.js, dependencies and the connectors for the storages
-you want to use in the HTML page header (note that more dependencies may be required
-depending on type of storages being used):
+The documentation can be found on [https://jio.nexedi.com/](https://jio.nexedi.com/)
 
-```html
-<script src="RSVP.js"></script>
-<script src="jio-latest.js"></script>
-```
-
-Then create your jIO instance like this:
-
-```javascript
-// create a new jio
-var jio_instance = jIO.createJIO({
-  type: "query",
-  sub_storage: {
-    type: "uuid",
-    sub_storage: {
-      "type": "indexeddb",
-      "database": "test"
-    }
-  }
-});
-```
-
-### Documents and Methods
-
-Documents are JSON strings that contain *metadata* (properties, like a filename)
-and *attachments* (optional content, for example *image.jpg*).
-
-jIO exposes the following methods to *create*, *read*, *update* and *delete* documents
-(for more information, including revision management and available options for
-each method, please refer to the documentation):
-
-```javascript
-// create and store new document
-jio_instance.post({"title": "some title"})
-  .then(function (new_id) {
-    ...
-  });
-
-// create or update an existing document
-jio_instance.put(new_id, {"title": "New Title"})
-  .then(function () 
-    // console.log("Document stored");
-  });
-
-// add an attachement to a document
-jio_instance.putAttachment(document_id, attachment_id, new Blob())
-  .then(function () {
-    // console.log("Blob stored");
-  });
-
-// read a document
-jio_instance.get(document_id)
-  .then(function (document) {
-    // console.log(document);
-    // {
-    //   "title": "New Title",
-    // }
-  });
-
-// read an attachement
-jio_instance.getAttachment(document_id, attachment_id)
-  .then(function (blob) {
-    // console.log(blob);
-  });
-
-// delete a document and its attachment(s)
-jio_instance.remove(document_id)
-  .then(function () {
-    // console.log("Document deleted");
-  });
-
-// delete an attachement
-jio_instance.removeAttachment(document_id, attachment_id)
-  .then(function () {
-    // console.log("Attachment deleted");
-  });
-
-// get all documents
-jio_instance.allDocs().then(function (response) {
-  // console.log(response);
-  // {
-  //   "data": {
-  //     "total_rows": 1,
-  //     "rows": [{
-  //       "id": "my_document",
-  //       "value": {}
-  //     }]
-  //   }
-  // }
-});
-```
-
-
-### Example
-
-This is an example of how to store a video file with one attachment in local
-storage. Note that attachments should be added after document creation.
-
-```javascript
-// create a new localStorage
-var jio_instance = jIO.createJIO({
-  "type": "local",
-});
-
-var my_video_blob = new Blob([my_video_binary_string], {
-  "type": "video/ogg"
-});
-
-// post the document
-jio_instance.put("myVideo", {
-  "title"       : "My Video",
-  "format"      : ["video/ogg", "vorbis", "HD"],
-  "language"    : "en",
-  "description" : "Images Compilation"
-}).then(function (response) {
-
-  // add video attachment
-  return jio_instance.putAttachment(
-    "myVideo",
-    "video.ogv",
-    my_video_blob
-  });
-
-}).then(function (response) {
-
-  alert('Video Stored');
-
-}, function (err) {
-
-  alert('Error when attaching the video');
-
-}, function (progression) {
-
-  console.log(progression);
-
-});
-```
-
-### Storage Locations
-
-jIO allows to build "storage trees" consisting of connectors to multiple
-storages (webDav, xWiki, S3, localStorage) and use type-storages to add features
-like revision management or indices to a child storage (sub_storage).
-
-The following storages are currently supported:
-
-- LocalStorage (browser local storage)
-- IndexedDB
-- ERP5Storage
-- DAVStorage (connect to webDAV, more information on the
-  [documentation](https://www.j-io.org/documentation/jio-documentation/))
-
-For more information on the specific storages including guidelines on how to
-create your own connector, please also refer to the [documentation](https://www.j-io.org/documentation/jio-documentation).
-
-### jIO Query
-
-jIO can use queries, which can be run in the allDocs() method to query document
-lists. A sample query would look like this (note that not all storages support
-allDocs and jio queries, and that pre-querying of documents on distant storages
-should best be done server-side):
-
-```javascript
-// run allDocs with query option on an existing jIO
-jio_instance.allDocs({
-  "query": '(fieldX: >= "string" AND fieldY: < "string")',
-  // records to display ("from to")
-  "limit": [0, 5],
-  // sort by
-  "sort_on": [[<string A>, 'descending']],
-  // fields to return in response
-  "select_list": [<string A>, <string B>]
-}).then(function (response) {
-  // console.log(response);
-  // {
-  //   "total_rows": 1,
-  //   "rows": [{
-  //     "id": <string>,
-  //     "value": {
-  //       <string A>: <string>,
-  //       <string B>: <string>
-  //     }
-  //   }, { .. }]
-  // }
-});
-```
-
-To find out more about queries, please refer to the documentation.
-
-### Authors
-
-- Francois Billioud
-- Tristan Cavelier
-- Sven Franck
-- Romain Courteaud
-
-### Copyright and license
-
-jIO is an open-source library and is licensed under the LGPL license. More
-information on LGPL can be found
-[here](http://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License).
-
-### Contribute
-
-Get development environment:
-
-
-    git clone https://lab.nexedi.com/nexedi/jio.git jio.git
-    cd jio.git
+### jIO Quickstart
+    git clone https://lab.nexedi.com/nexedi/jio.git
     npm install
-    alias grunt="./node_modules/grunt-cli/bin/grunt"
-    grunt
-
-
-Run tests:
-
-
     grunt server
 
 
-and open http://127.0.0.1:9000/test/tests.html
+### jIO Code
+
+RenderJS source code is hosted on Gitlab at [https://lab.nexedi.com/nexedi/jio](https://lab.nexedi.com/nexedi/jio) (Github [mirror](https://github.com/nexedi/jio/) - please use the issue tracker on Gitlab)
 
-Submit merge requests on lab.nexedi.com.
+### jIO Test
+You can run tests after installing and building jIO by opening the */test/* folder

package.json

 {
   "name": "jio",
-  "version": "v3.21.0",
+  "version": "v3.23.1",
   "license": "LGPLv3",
   "author": "Nexedi SA",
   "contributors": [

src/jio.storage/documentstorage.js

 /*jslint nomen: true*/
-/*global Blob, atob, btoa, RSVP*/
-(function (jIO, Blob, atob, btoa, RSVP) {
+/*global Blob, RSVP, unescape, escape*/
+(function (jIO, Blob, RSVP, unescape, escape) {
   "use strict";
-
   /**
    * The jIO DocumentStorage extension
    *
@@ -18,7 +17,13 @@
   var DOCUMENT_EXTENSION = ".json",
     DOCUMENT_REGEXP = new RegExp("^jio_document/([\\w=]+)" +
                                  DOCUMENT_EXTENSION + "$"),
-    ATTACHMENT_REGEXP = new RegExp("^jio_attachment/([\\w=]+)/([\\w=]+)$");
+    ATTACHMENT_REGEXP = new RegExp("^jio_attachment/([\\w=]+)/([\\w=]+)$"),
+    btoa = function (str) {
+      return window.btoa(unescape(encodeURIComponent(str)));
+    },
+    atob = function (str) {
+      return decodeURIComponent(escape(window.atob(str)));
+    };
 
   function getSubAttachmentIdFromParam(id, name) {
     if (name === undefined) {
@@ -225,4 +230,4 @@
 
   jIO.addStorage('document', DocumentStorage);
 
-}(jIO, Blob, atob, btoa, RSVP));
+}(jIO, Blob, RSVP, unescape, escape));

src/jio.storage/fbstorage.js

+/*jslint nomen: true */
+/*global RSVP, UriTemplate*/
+(function (jIO, RSVP, UriTemplate) {
+  "use strict";
+
+  var GET_POST_URL = "https://graph.facebook.com/v2.9/{+post_id}" +
+      "?fields={+fields}&access_token={+access_token}",
+    get_post_template = UriTemplate.parse(GET_POST_URL),
+    GET_FEED_URL = "https://graph.facebook.com/v2.9/{+user_id}/feed" +
+        "?fields={+fields}&limit={+limit}&since={+since}&access_token=" +
+        "{+access_token}",
+    get_feed_template = UriTemplate.parse(GET_FEED_URL);
+
+  function FBStorage(spec) {
+    if (typeof spec.access_token !== 'string' || !spec.access_token) {
+      throw new TypeError("Access Token must be a string " +
+                          "which contains more than one character.");
+    }
+    if (typeof spec.user_id !== 'string' || !spec.user_id) {
+      throw new TypeError("User ID must be a string " +
+                          "which contains more than one character.");
+    }
+    this._access_token = spec.access_token;
+    this._user_id = spec.user_id;
+    this._default_field_list = spec.default_field_list || [];
+    this._default_limit = spec.default_limit || 500;
+  }
+
+  FBStorage.prototype.get = function (id) {
+    var that = this;
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: get_post_template.expand({post_id: id,
+            fields: that._default_field_list, access_token: that._access_token})
+        });
+      })
+      .push(function (result) {
+        return JSON.parse(result.target.responseText);
+      });
+  };
+
+  function paginateResult(url, result, select_list) {
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: url
+        });
+      })
+      .push(function (response) {
+        return JSON.parse(response.target.responseText);
+      },
+        function (err) {
+          throw new jIO.util.jIOError("Getting feed failed " + err.toString(),
+            err.target.status);
+        })
+      .push(function (response) {
+        if (response.data.length === 0) {
+          return result;
+        }
+        var i, j, obj = {};
+        for (i = 0; i < response.data.length; i += 1) {
+          obj.id = response.data[i].id;
+          obj.value = {};
+          for (j = 0; j < select_list.length; j += 1) {
+            obj.value[select_list[j]] = response.data[i][select_list[j]];
+          }
+          result.push(obj);
+          obj = {};
+        }
+        return paginateResult(response.paging.next, result, select_list);
+      });
+  }
+
+  FBStorage.prototype.buildQuery = function (query) {
+    var that = this, fields = [], limit = this._default_limit,
+      template_argument = {
+        user_id: this._user_id,
+        limit: limit,
+        access_token: this._access_token
+      };
+    if (query.include_docs) {
+      fields = fields.concat(that._default_field_list);
+    }
+    if (query.select_list) {
+      fields = fields.concat(query.select_list);
+    }
+    if (query.limit) {
+      limit = query.limit[1];
+    }
+    template_argument.fields = fields;
+    template_argument.limit = limit;
+    return paginateResult(get_feed_template.expand(template_argument), [],
+      fields)
+      .push(function (result) {
+        if (!query.limit) {
+          return result;
+        }
+        return result.slice(query.limit[0], query.limit[1]);
+      });
+  };
+
+  FBStorage.prototype.hasCapacity = function (name) {
+    var this_storage_capacity_list = ["list", "select", "include", "limit"];
+    if (this_storage_capacity_list.indexOf(name) !== -1) {
+      return true;
+    }
+  };
+
+  jIO.addStorage('facebook', FBStorage);
+
+}(jIO, RSVP, UriTemplate));
\ No newline at end of file

src/jio.storage/indexeddbstorage.js

@@ -385,7 +385,9 @@
               end_index -= 1;
             }
             function resolver(result) {
-              result_list.push(result);
+              if (result.blob !== undefined) {
+                result_list.push(result);
+              }
               resolve(result_list);
             }
             function getPart(i) {

src/queries/query.js

@@ -416,7 +416,7 @@
     }
     return new RegExp("^" + stringEscapeRegexpCharacters(string)
       .replace(regexp_percent, '[\\s\\S]*')
-      .replace(regexp_underscore, '.') + "$");
+      .replace(regexp_underscore, '.') + "$", "i");
   }
 
   /**
@@ -743,7 +743,7 @@
       value = '%' + this.value + '%';
       for (k in item) {
         if (item.hasOwnProperty(k)) {
-          if (k !== '__id') {
+          if (k !== '__id' && item[k]) {
             if (matchMethod(item[k], value) === true) {
               return true;
             }

test/jio.storage/documentstorage.tests.js

@@ -236,6 +236,46 @@
       });
   });
 
+  test("with special utf-8 char", function () {
+    stop();
+    expect(5);
+
+    Storage200.prototype.putAttachment = function (id, name, blob) {
+      equal(blob.type, "application/json", "Blob type is OK");
+      equal(id, "foo", "putAttachment 200 called");
+      equal(
+        name,
+        "jio_document/Zm9vw6kKYmFy5rWL6K+V5Zub8J+YiA==.json",
+        "putAttachment 200 called"
+      );
+
+      return jIO.util.readBlobAsText(blob)
+        .then(function (result) {
+          deepEqual(JSON.parse(result.target.result),
+                    {"title": "fooé\nbar测试四😈"},
+                    "JSON is in blob");
+          return id;
+        });
+    };
+    var jio = jIO.createJIO({
+      type: "document",
+      document_id: "foo",
+      sub_storage: {
+        type: "documentstorage200"
+      }
+    });
+
+    jio.put("fooé\nbar测试四😈", {"title": "fooé\nbar测试四😈"})
+      .then(function (result) {
+        equal(result, "fooé\nbar测试四😈");
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
   /////////////////////////////////////////////////////////////////
   // documentStorage.remove
   /////////////////////////////////////////////////////////////////

test/jio.storage/indexeddbstorage.tests.js

@@ -1357,6 +1357,33 @@
       });
   });
 
+  test("retrieve empty blob", function () {
+    var context = this,
+      attachment = "attachment",
+      blob = new Blob();
+    stop();
+    expect(1);
+
+    deleteIndexedDB(context.jio)
+      .then(function () {
+        return context.jio.put("foo", {"title": "bar"});
+      })
+      .then(function () {
+        return context.jio.putAttachment("foo", attachment, blob);
+      })
+      .then(function () {
+        return context.jio.getAttachment("foo", attachment);
+      })
+      .then(function (result) {
+        deepEqual(result, blob, "check empty blob");
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
   /////////////////////////////////////////////////////////////////
   // indexeddbStorage.removeAttachment
   /////////////////////////////////////////////////////////////////

test/queries/key.tests.js

@@ -601,7 +601,8 @@
         then(function (dl) {
           deepEqual(dl, [
             {'identifier': 'àéîöùç'},
-            {'identifier': 'âèî ôùc'}
+            {'identifier': 'âèî ôùc'},
+            {'identifier': 'ÀÉÎÖÙÇ'}
           ], 'It should be possible to query regardless of accents');
         })
     );

test/queries/tests.js

@@ -440,7 +440,72 @@
       });
   });
 
-// Asterisk wildcard is not supported yet.
+  // Test queries which have components with key and without it.
+  // Default operator used if not present is AND.
+  test('Mixed query', function () {
+    var doc_list = [
+      {"identifier": "a", "value": "test1", "time": "2016"},
+      {"identifier": "b", "value": "test2", "time": "2017"},
+      {"identifier": "c", "value": "test3", "time": "2017"}
+    ];
+    stop();
+    expect(2);
+    jIO.QueryFactory.create('test2 time:%2017%').exec(doc_list).
+      then(function (doc_list) {
+        deepEqual(doc_list, [
+          {"identifier": "b", "value": "test2", "time": "2017"}
+        ], 'Document with test2 in any column and 2017 in time is matched');
+
+        doc_list = [
+          {"identifier": "a", "value": "test post 1", "time": "2016"},
+          {"identifier": "b", "value": "test post 2", "time": "2017"},
+          {"identifier": "c", "value": "test post 3", "time": "2018"}
+        ];
+
+        return jIO.QueryFactory.create('value:"%test post 2%" OR c OR ' +
+          '2016').exec(doc_list).
+          then(function (doc_list) {
+            deepEqual(doc_list, [
+              {"identifier": "a", "value": "test post 1", "time": "2016"},
+              {"identifier": "b", "value": "test post 2", "time": "2017"},
+              {"identifier": "c", "value": "test post 3", "time": "2018"}
+            ], 'Documents with "test post 2" in value or "c" or "2016" ' +
+              'anywhere are matched');
+          }).always(start);
+      });
+  });
+
+  test('Case insensitive queries', function () {
+    var doc_list = [
+      {"identifier": "a", "value": "Test Post", "time": "2016"},
+      {"identifier": "b", "value": "test post", "time": "2017"},
+      {"identifier": "c", "value": "test3", "time": "2018"}
+    ];
+    stop();
+    expect(2);
+    jIO.QueryFactory.create('test post').exec(doc_list).
+      then(function (doc_list) {
+        deepEqual(doc_list, [
+          {"identifier": "a", "value": "Test Post", "time": "2016"},
+          {"identifier": "b", "value": "test post", "time": "2017"}
+        ], 'Documunts with the value irrespective of case are matched');
+
+        doc_list = [
+          {"identifier": "a", "value": "Test Post", "time": "2016"},
+          {"identifier": "b", "value": "test post", "time": "2017"},
+          {"identifier": "c", "value": "test3", "time": "2018"}
+        ];
+
+        return jIO.QueryFactory.create('value:"test post"').exec(doc_list).
+          then(function (doc_list) {
+            deepEqual(doc_list, [
+              {"identifier": "b", "value": "test post", "time": "2017"}
+            ], 'If value is in quotes, only match if exactly same');
+          }).always(start);
+      });
+  });
+
+  // Asterisk wildcard is not supported yet.
 /*  test('Full text query with asterisk', function () {
     var doc_list = [
       {"identifier": "abc"},

test/tests.html

@@ -52,6 +52,7 @@
   <script src="jio.storage/zipstorage.tests.js"></script>
   <script src="jio.storage/gdrivestorage.tests.js"></script>
   <script src="jio.storage/websqlstorage.tests.js"></script>
+  <script src="jio.storage/fbstorage.tests.js"></script>
   <!--script src="../lib/jquery/jquery.min.js"></script>
   <script src="../src/jio.storage/xwikistorage.js"></script>
   <script src="jio.storage/xwikistorage.tests.js"></script-->