Node: build a NodeJS compatible version.

Implement a basic FormData support, used to serialize the POST body.

Use xhr2 library to replicate XHMLHttpRequest.

Desactivate all broken storages (IndexedDB, localstorage, webdav, ...) which rely on browser API.

Thanks to Aurelien Calonne <aurel@nexedi.com> and Guillaume Royer <guillaume.royer@clearroad.io> for their contributions.

Makefile

@@ -29,6 +29,8 @@ EXTERNALDIR = external
 VERSION = 3.32.1
 JIOVERSION = ${DISTDIR}/jio-v${VERSION}.js
 JIOLATEST = ${DISTDIR}/jio-latest.js
+JIONODEVERSION = ${DISTDIR}/jio-v${VERSION}-node.js
+JIONODELATEST = ${DISTDIR}/jio-latest-node.js
 
 all: fetch lint build
 
@@ -77,6 +79,21 @@ ${LINTDIR}/${EXAMPLEDIR}/%.js: ${EXAMPLEDIR}/%.js
 	${JSLINT} ${LINTOPTS} --predef RSVP --predef window --predef QUnit --predef jIO --predef rJS $<
 	@cat $< > $@
 
+${LINTDIR}/node/%.js: ${SRCDIR}/node/%.js
+	@mkdir -p $(@D)
+	${JSLINT} ${LINTOPTS} --nomen true $<
+	@cat $< > $@
+
+${LINTDIR}/${TESTDIR}/%.js: ${TESTDIR}/%.js
+	@mkdir -p $(@D)
+	${JSLINT} ${LINTOPTS} $<
+	@cat $< > $@
+
+${LINTDIR}/${TESTDIR}/node/%.js: ${TESTDIR}/node/%.js
+	@mkdir -p $(@D)
+	${JSLINT} ${LINTOPTS} $<
+	@cat $< > $@
+
 #############################################
 # Check test files
 #############################################
@@ -88,12 +105,15 @@ lint: $(patsubst ${TESTDIR}/jio.storage/%.js, ${LINTDIR}/${TESTDIR}/jio.storage/
 	${LINTDIR}/queries/query.js \
 	${LINTDIR}/jio.date/jiodate.js \
 	${LINTDIR}/jio.js \
+	${LINTDIR}/node/jio.js \
+	${LINTDIR}/${TESTDIR}/node.js \
+	${LINTDIR}/${TESTDIR}/node/node-require.js \
 	$(patsubst ${SRCDIR}/jio.storage/%.js, ${LINTDIR}/jio.storage/%.js, $(wildcard ${SRCDIR}/jio.storage/*.js))
 
 #############################################
 # Build
 #############################################
-build: ${JIOLATEST}
+build: ${JIOLATEST} ${JIONODELATEST}
 
 ${JIOLATEST}: ${JIOVERSION}
 	@mkdir -p $(@D)
@@ -134,6 +154,45 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
 	@mkdir -p $(@D)
 	cat $^ > $@
 
+#############################################
+# Node
+#############################################
+${JIONODELATEST}: ${JIONODEVERSION}
+	@mkdir -p $(@D)
+	cp $< $@
+
+${JIONODEVERSION}: ${SRCDIR}/node/jio-start.js \
+	${EXTERNALDIR}/rsvp-2.0.4.js \
+	${EXTERNALDIR}/moment.js \
+	${EXTERNALDIR}/URI.js \
+	${EXTERNALDIR}/uritemplate.js \
+	${EXTERNALDIR}/rusha.js \
+	${SRCDIR}/node/jio-external.js \
+	${EXTERNALDIR}/xhr2.js \
+	${SRCDIR}/queries/parser-begin.js \
+	${SRCDIR}/queries/build/parser.js \
+	${SRCDIR}/queries/parser-end.js \
+	${SRCDIR}/queries/query.js \
+	${SRCDIR}/node/jio-compat.js \
+	${SRCDIR}/jio.date/jiodate.js \
+	${SRCDIR}/jio.js \
+	${SRCDIR}/node/jio.js \
+	${SRCDIR}/jio.storage/replicatestorage.js \
+	${SRCDIR}/jio.storage/shastorage.js \
+	${SRCDIR}/jio.storage/uuidstorage.js \
+	${SRCDIR}/jio.storage/memorystorage.js \
+	${SRCDIR}/jio.storage/dropboxstorage.js \
+	${SRCDIR}/jio.storage/gdrivestorage.js \
+	${SRCDIR}/jio.storage/unionstorage.js \
+	${SRCDIR}/jio.storage/erp5storage.js \
+	${SRCDIR}/jio.storage/querystorage.js \
+	${SRCDIR}/jio.storage/drivetojiomapping.js \
+	${SRCDIR}/jio.storage/documentstorage.js \
+	${SRCDIR}/jio.storage/fbstorage.js \
+	${SRCDIR}/node/jio-end.js
+	@mkdir -p $(@D)
+	cat $^ > $@
+
 #############################################
 # Jison
 #############################################
@@ -190,7 +249,7 @@ ${EXTERNALDIR}/renderjs-latest.js:
 	@mkdir -p $(@D)
 	curl -s -o $@ https://lab.nexedi.com/nexedi/renderjs/raw/master/dist/renderjs-latest.js
 
-.PHONY: clean ${JIOVERSION}
+.PHONY: clean ${JIOVERSION} ${JIONODEVERSION}
 
 clean:
 	rm -rf ${LINTDIR}

README.md

@@ -9,7 +9,7 @@ The documentation can be found on [https://jio.nexedi.com/](https://jio.nexedi.c
 ### jIO Quickstart
     git clone https://lab.nexedi.com/nexedi/jio.git
     cd jio.git
-    npm install jslint@0.9.2 jison@0.4.16
+    npm install jslint@0.9.2 jison@0.4.16 git://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3
     make
 
 

src/node/jio-compat.js

+/*
+ * Copyright 2018, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+
+/*global window, WeakMap, ArrayBuffer, Uint8Array */
+(function (window, WeakMap, ArrayBuffer, Uint8Array) {
+  "use strict";
+
+  var html5weakmap = new WeakMap();
+
+  function EventTarget() {
+    html5weakmap.set(this, Object.create(null));
+  }
+
+  EventTarget.prototype.addEventListener = function (type, listener) {
+    if (typeof listener !== 'function') {
+      return;
+    }
+    var em = html5weakmap.get(this);
+    type = type.toString();
+    if (em[type]) {
+      em[type].push(listener);
+    } else {
+      em[type] = [listener];
+    }
+  };
+
+  EventTarget.prototype.removeEventListener = function (type, listener) {
+    if (typeof listener !== 'function') {
+      return;
+    }
+    var em = html5weakmap.get(this),
+      i,
+      listeners = em[type];
+
+    type = type.toString();
+
+    if (listeners) {
+      for (i = 0; i < listeners.length; i += 1) {
+        if (listeners[i] === listener) {
+          if (listeners.length === 1) {
+            delete em[type];
+            return;
+          }
+          listeners.splice(i, 1);
+          return;
+        }
+      }
+    }
+  };
+
+  EventTarget.prototype.dispatchEvent = function (event) {
+    var type = event.type.toString(),
+      em = html5weakmap.get(this),
+      ontype = 'on' + type,
+      i,
+      listeners;
+
+    if (typeof this[ontype] === 'function') {
+      try {
+        this[ontype](event);
+      } catch (ignore) {}
+    }
+    listeners = em[type];
+    if (listeners) {
+      for (i = 0; i < listeners.length; i += 1) {
+        try {
+          listeners[i](event);
+        } catch (ignore) {}
+      }
+    }
+  };
+
+  window.EventTarget = window.EventTarget || EventTarget;
+
+  function Blob(blobParts, options) {
+    // https://developer.mozilla.org/en-US/docs/Web/API/Blob
+    var i,
+      priv = {},
+      buffers = [];
+
+    html5weakmap.set(this, priv);
+    if (blobParts) {
+      for (i = 0; i < blobParts.length; i += 1) {
+        if (Buffer.isBuffer(blobParts[i])) {
+          buffers.push(blobParts[i]);
+        } else if (blobParts[i] instanceof Blob) {
+          buffers.push(html5weakmap.get(blobParts[i]).data);
+        } else if (blobParts[i] instanceof ArrayBuffer) {
+          buffers.push(new Buffer(new Uint8Array(blobParts[i])));
+        } else {
+          buffers.push(new Buffer(String(blobParts[i])));
+        }
+      }
+    }
+    priv.data = Buffer.concat(buffers);
+    Object.defineProperty(this, 'size', {
+      enumerable: true,
+      value: priv.data.length
+    });
+    Object.defineProperty(this, 'type', {
+      enumerable: true,
+      value: options ? String(options.type || '') : ''
+    });
+  }
+
+  Blob.prototype.size = 0;
+  Blob.prototype.type = '';
+  Blob.prototype.slice = function (start, end, contentType) {
+    return new Blob([html5weakmap.get(this).data.slice(start, end)], {
+      type: contentType
+    });
+  };
+
+  window.Blob = Blob;//window.Blob || Blob;
+
+  function FileReader() {
+    EventTarget.call(this);
+  }
+
+  FileReader.prototype = Object.create(EventTarget.prototype);
+  Object.defineProperty(FileReader, 'constructor', {
+    value: FileReader
+  });
+
+  FileReader.prototype.readAsText = function (blob) {
+    var target = this,
+      priv = html5weakmap.get(blob),
+      result = priv.data.toString(),
+      event = Object.freeze({
+        type: 'load',
+        target: target
+      });
+
+    process.nextTick(function () {
+      target.result = result;
+      target.dispatchEvent(event);
+    });
+  };
+
+  FileReader.prototype.readAsArrayBuffer = function (blob) {
+    var target = this,
+      priv = html5weakmap.get(blob),
+      result = new Uint8Array(priv.data).buffer,
+      event = Object.freeze({
+        type: 'load',
+        target: target
+      });
+
+    process.nextTick(function () {
+      target.result = result;
+      target.dispatchEvent(event);
+    });
+  };
+
+  FileReader.prototype.readAsDataURL = function (blob) {
+    var target = this,
+      priv = html5weakmap.get(blob),
+      result = 'data:' + blob.type + ';base64,' + priv.data.toString('base64'),
+      event = Object.freeze({
+        type: 'load',
+        target: target
+      });
+
+    process.nextTick(function () {
+      target.result = result;
+      target.dispatchEvent(event);
+    });
+  };
+
+  window.FileReader = window.FileReader || FileReader;
+
+  function atob(str) {
+    try {
+      return window.atob(str);
+    } catch (err) {
+      var buffer;
+      if (str instanceof Buffer) {
+        buffer = str;
+      } else {
+        buffer = Buffer.from(str.toString(), 'base64');
+      }
+      return buffer.toString('binary');
+    }
+  }
+
+  window.atob = window.atob || atob;
+
+  function btoa(str) {
+    try {
+      return window.btoa(str);
+    } catch (err) {
+      var buffer;
+      if (str instanceof Buffer) {
+        buffer = str;
+      } else {
+        buffer = Buffer.from(str.toString(), 'binary');
+      }
+      return buffer.toString('base64');
+    }
+  }
+
+  window.btoa = window.btoa || btoa;
+
+}(window, WeakMap, ArrayBuffer, Uint8Array));
+
+var XMLHttpRequest = global.XMLHttpRequest || module.exports,
+  Blob = window.Blob,
+  atob = window.atob,
+  btoa = window.btoa,
+  FileReader = window.FileReader,
+  QueryFactory = window.QueryFactory,
+  Query = window.Query,
+  SimpleQuery = window.SimpleQuery,
+  ComplexQuery = window.ComplexQuery;
+

src/node/jio-end.js

+
+  module = node_module;
+
+  jIO.node_env = window;
+  module.exports = jIO;
+
+} ({}));
\ No newline at end of file

src/node/jio-external.js

+  var RSVP = window.RSVP,
+    moment = global.moment,
+    UriTemplate = window.UriTemplate,
+    Rusha = window.Rusha;
+
+  // Allow xhr2 to export XMLHttpRequest
+  module = {};

src/node/jio-start.js

+(function (define, exports) {
+  var navigator = null,
+    window = {},
+    node_module = module;
+
+  module = undefined;

src/node/jio.js

+/*
+ * Copyright 2018, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+
+/*global window */
+(function (window, jIO, Blob) {
+  "use strict";
+
+  var FormData,
+    originalAjax;
+
+  // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/
+  // Using_XMLHttpRequest#Submitting_forms_and_uploading_files
+  FormData = function FormData() {
+    this.boundary = "---------------------------" + Date.now().toString(16);
+    this.body = '';
+  };
+  FormData.prototype.append = function (name, value, filename) {
+    this.body += '--' + this.boundary +
+                 '\r\nContent-Disposition: form-data; name="' + name;
+    if (filename !== undefined) {
+      this.body += '"; filename="' + filename;
+    }
+    this.body += '"\r\n\r\n' + value + '\r\n';
+  };
+  window.FormData = FormData;
+
+  originalAjax = jIO.util.ajax;
+  jIO.util.ajax = function ajax(param) {
+    if (param.data instanceof Blob) {
+      // Blob is not supported by xhr2, so convert to ArrayBuffer instead
+      return jIO.util.readBlobAsArrayBuffer(param.data).then(function (data) {
+        param.data = data.target.result;
+        return originalAjax(param);
+      });
+    }
+
+    if (param.data instanceof FormData) {
+      // Implement minimal FormData for erp5storage
+      if (!param.hasOwnProperty('headers')) {
+        param.headers = {};
+      }
+      param.headers["Content-Type"] = "multipart\/form-data; boundary=" +
+                                      param.data.boundary;
+      param.data.body += '--' + param.data.boundary + '--\r\n';
+      param.data = param.data.body;
+      return originalAjax(param);
+    }
+
+    return originalAjax(param);
+  };
+
+}(window, window.jIO, window.Blob));
+
+// Define a global variable to allow storages to access jIO
+var jIO = window.jIO,
+  FormData = window.FormData,
+  jiodate = window.jiodate;

test/node.js

+/*
+ * Copyright 2018, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+/*global require, process, console, JSON */
+(function (require, process, console, JSON) {
+  "use strict";
+
+  var testrunner = require('qunit'),
+    report_data = process.env.CI ? {} : null,
+    runner_options = process.env.CI ? {
+      log: {}
+    } : {
+      log: {
+        errors: true,
+        summary: true,
+        tests: true
+      }
+    };
+
+  testrunner.setup(runner_options);
+
+  if (report_data) {
+    testrunner.log.add = function (type, obj) {
+      if (!report_data.hasOwnProperty(type)) {
+        report_data[type] = [];
+      }
+      report_data[type].push(obj);
+      return report_data[type];
+    };
+  }
+
+  testrunner.run({
+    code: 'test/node/node-require.js',
+    tests: [
+      'test/jio/util.js',
+      'test/queries/key.tests.js',
+      'test/queries/key-schema.tests.js',
+      /*
+      'test/queries/tests.js',
+      */
+      'test/queries/key-typechecks.tests.js',
+      'test/queries/jiodate.tests.js',
+      'test/queries/key-jiodate.tests.js',
+      'test/jio.storage/documentstorage.tests.js',
+      'test/jio.storage/drivetojiomapping.tests.js',
+      'test/jio.storage/dropboxstorage.tests.js',
+      'test/jio.storage/erp5storage.tests.js',
+      'test/jio.storage/fbstorage.tests.js',
+      'test/jio.storage/gdrivestorage.tests.js',
+      'test/jio.storage/memorystorage.tests.js',
+      'test/jio.storage/querystorage.tests.js',
+      'test/jio.storage/replicatestorage.tests.js',
+      'test/jio.storage/replicatestorage_fastrepair.tests.js',
+      'test/jio.storage/replicatestorage_fastrepairattachment.tests.js',
+      'test/jio.storage/replicatestorage_repair.tests.js',
+      'test/jio.storage/replicatestorage_repairattachment.tests.js',
+      'test/jio.storage/shastorage.tests.js',
+      'test/jio.storage/unionstorage.tests.js',
+      'test/jio.storage/uuidstorage.tests.js'
+    ]
+  }, function (err) {
+    if (err) {
+      console.error('error', err);
+    }
+    if (report_data) {
+      console.log(JSON.stringify(report_data));
+    }
+  });
+
+}(require, process, console, JSON));

test/node/node-require.js

+/*
+ * Copyright 2018, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+/*global require, global, Object */
+(function (require, global, Object) {
+  "use strict";
+
+  var sinon = require('./sinon-require'),
+    jIO;
+  global.sinon = sinon;
+  global.XMLHttpRequest = sinon.FakeXMLHttpRequest;
+
+  jIO = require('../../dist/jio-latest-node');
+  global.jIO = jIO;
+  Object.keys(jIO.node_env).forEach(function (key) {
+    global[key] = jIO.node_env[key];
+  });
+
+}(require, global, Object));

[Hamza]

Hello @romain , Today I built the Jio project but the npm couldn't install the package from git directly, I mean it couldn't find git://github.com/qunitjs/node-qunit.git#v0.9.3 and in the other hand it shows a warning that there is a high severity vulnerability, to remove it I installed jslint@0.12.1 and it worked fine in my machine (without git://github.com/qunitjs/node-qunit.git#v0.9.3 ) Please Check Bests, Hamza

[Romain Courteaud]

Do you use a web runner to develop jio? If not, can you try installing the jstestnode software release on a web runner?

Don't you get error when running the Makefile with jslint@0.12.1 (which I think enforce es6 conventions)?