Skip to content

J
jio
Commit 61749ec2 authored by preetwinder's avatar preetwinder
Browse files
Options

use common functions for indexeddb

parent 22094762
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 490 additions and 799 deletions
Makefile
View file @ 61749ec2
......@@ -59,6 +59,11 @@ ${LINTDIR}/queries/query.js: ${SRCDIR}/queries/query.js
${JSLINT} ${LINTOPTS} $<
@cat $< > $@
${LINTDIR}/indexeddb/indexeddb.js: ${SRCDIR}/indexeddb/indexeddb.js
@mkdir -p $(@D)
${JSLINT} ${LINTOPTS} $<
@cat $< > $@
${LINTDIR}/${TESTDIR}/jio.storage/%.js: ${TESTDIR}/jio.storage/%.js
@mkdir -p $(@D)
${JSLINT} ${LINTOPTS} --predef QUnit --predef RSVP --predef jIO $<
......@@ -104,6 +109,7 @@ lint: $(patsubst ${TESTDIR}/jio.storage/%.js, ${LINTDIR}/${TESTDIR}/jio.storage/
$(patsubst ${EXAMPLEDIR}/%.js, ${LINTDIR}/${EXAMPLEDIR}/%.js, $(wildcard ${EXAMPLEDIR}/*.js)) \
${LINTDIR}/queries/query.js \
${LINTDIR}/jio.date/jiodate.js \
${LINTDIR}/indexeddb/indexeddb.js \
${LINTDIR}/jio.js \
${LINTDIR}/node/jio.js \
${LINTDIR}/${TESTDIR}/node.js \
......@@ -129,6 +135,7 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
${SRCDIR}/queries/parser-end.js \
${SRCDIR}/queries/query.js \
${SRCDIR}/jio.date/jiodate.js \
${SRCDIR}/indexeddb/indexeddb.js \
${SRCDIR}/jio.js \
${EXTERNALDIR}/rusha.js \
${SRCDIR}/jio.storage/replicatestorage.js \
......
examples/index2_benchmark.html deleted 100644 → 0
View file @ 22094762
<!DOCTYPE html>
<!--
Copyright 2019, Nexedi SA
This program is free software: you can Use, Study, Modify and Redistribute
it under the terms of the GNU General Public License version 3, or (at your
option) any later version, as published by the Free Software Foundation.
You can also Link and Combine this program with other software covered by
the terms of any of the Free Software licenses or any of the Open Source
Initiative approved licenses and Convey the resulting work. Corresponding
source of such a combination shall include the source code for all other
software used.
This program is distributed WITHOUT ANY WARRANTY; without even the implied
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See COPYING file for full licensing terms.
See https://www.nexedi.com/licensing for rationale and options.
-->
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>jIO Query Performance test</title>
<script src="../external/rsvp-2.0.4.js"></script>
<script src="../external/renderjs-latest.js"></script>
<script src="../dist/jio-latest.js"></script>
<!-- <link rel="stylesheet" href="../external/qunit.css" type="text/css" media="screen"/> -->
<!-- <script src="../external/qunit.js" type="text/javascript"></script> -->
<script src="index2_benchmark.js"></script>
</head>
<body>
<h1> Testing index2 query </h1>
<a>Test</a>
<p></p>
<div></div>
</body>
</html>
examples/index2_benchmark.js deleted 100644 → 0
View file @ 22094762
/*global performance, String*/
(function (window, jIO, rJS) {
"use strict";
var test_count = 15;
/*function randomi(limit) {
return Math.floor(Math.random() * Math.floor(limit));
}
function randomSentence(length) {
var alphabet = ['a', 'b', 'c', 'd', 'e', ' ', 'f', 'g', 'h', 'i', 'j', 'k',
' ', 'l', 'm', 'n', 'o', ' ', 'p', 'q', 'r', 's', 't', ' ',
'u', 'v', 'w', ' ', 'x', 'y', 'z', ' '], sentence = '', z;
for (z = 0; z < length; z += 1) {
sentence += alphabet[randomi(alphabet.length - 1)];
}
return sentence;
}
function randomSentenceArray(sentence_length, array_length) {
var y, sentence_array = [];
for (y = 0; y < array_length; y += 1) {
sentence_array.push(randomSentence(sentence_length));
}
return sentence_array;
}*/
function get_fake_data_values2(i) {
if (i === 0 || i === 1 || i === 2) {
return {'url': 'renderjs.com', 'name': 'erp5', 'user': 'preet'};
}
if (i === 3 || i === 4) {
return {'url': 'erp5.com', 'name': 'erp5', 'user': 'test'};
}
if (i === 5 || i === 6 || i === 7) {
return {'url': 'nexedi.com', 'name': 'nexedi', 'user': 'prabetcder'};
}
if (i === 10 || i === 11) {
return {'url': 'vifib.com', 'name': 'renderjs', 'user': 'preetwinder'};
}
if (i === 12 || i === 13) {
return {'url': 'renderjs.com', 'name': 'jio', 'user': 'obscure'};
}
return {'url': 'jio.nexedi.com', 'name': 'jio', 'user': 'praounsteter'};
}
/*function get_fake_data_values(i) {
var data_value = {
'id': i,
'url': 'https://streetsite.com/profiles/' + i,
'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i,
'short_description': randomSentence(10 + randomi(40)),
'description': randomSentence(randomi(250)),
'comments': randomSentenceArray(randomi(500), randomi(20))
};
if (i === 9900) {
data_value.short_description = 'test';
}
if (i === 7500) {
data_value.short_description = 'preet';
}
if (i === 5400) {
data_value.short_description = 'obscure';
}
if (i === 3200) {
data_value.short_description = 'precise';
}
if (i === 1200) {
data_value.short_description = 'environ';
}
return data_value;
}*/
/* function sequential_test(i, storage) {
if (i < test_count) {
var data_value = {
'id': i,
'url': 'https://streetsite.com/profiles/' + i,
'pic_url': 'https://cdn.streetsite.com/pictures/saoteuhcu/' + i,
'short_description': randomSentence(10 + randomi(40)),
'description': randomSentence(randomi(250)),
'comments': randomSentenceArray(randomi(500), randomi(20))
};
if (i === 99000) {
data_value.short_description = 'test';
}
if (i % 100 === 0) {
data_value.short_description = 'preet';
}
if (i % 1000 === 0) {
data_value.short_description = 'obscure';
}
if (i === 32000) {
data_value.short_description = 'precise precise precise';
}
if (i === 120000) {
data_value.short_description = 'environ';
}
return storage.put(String(i), data_value)
.then(function () {
if (i % 1000 === 0) {
console.log(i);
}
data_value = null;
return sequential_test(i + 1, storage);
});
}
return;
}*/
rJS(window)
.declareService(function () {
var storage = jIO.createJIO({
type: "index2",
database: "index2test2",
index_keys: ["user", "name", "url"],
sub_storage: {
type: "indexeddb",
database: "index2testdata2",
}
}), promise_list = [], i, time;
console.log('Staring to write ' + test_count + ' documents');
//sequential_test(0, storage);
for (i = 0; i < test_count; i += 1) {
promise_list.push(storage.put(String(i), get_fake_data_values2(i)));
}
promise_list.push(storage.put('325', get_fake_data_values2(325)));
time = performance.now();
return RSVP.all(promise_list)
.then(function () {
console.log('Time to write - ', (performance.now() - time));
console.log('Starting queries');
console.log('Query 1');
time = performance.now();
return storage.allDocs({query: "user:preetwinder"});
})
.then(function (result) {
console.log('Time to query 1 - ', (performance.now() - time));
console.log(result);
console.log('Query 2');
time = performance.now();
return storage.allDocs({query: 'user:preet'});
})
.then(function (result) {
console.log('Time to query 2 - ', (performance.now() - time));
console.log(result);
console.log('Query 3');
time = performance.now();
return storage.allDocs({query: "(name:jio OR url:nexedi.com" +
") AND user:obscure"});
})
.then(function (result) {
console.log('Time to query 3 - ', performance.now() - time);
console.log(result);
console.log('Query 4');
time = performance.now();
return storage.allDocs({query: 'name:not'});
})
.then(function (result) {
console.log('Time to query 4 - ', performance.now() - time);
console.log(result);
});
});
}(window, jIO, rJS));
\ No newline at end of file
src/indexeddb/indexeddb.js 0 → 100644
View file @ 61749ec2
/*
* Copyright 2019, Nexedi SA
*
* This program is free software: you can Use, Study, Modify and Redistribute
* it under the terms of the GNU General Public License version 3, or (at your
* option) any later version, as published by the Free Software Foundation.
*
* You can also Link and Combine this program with other software covered by
* the terms of any of the Free Software licenses or any of the Open Source
* Initiative approved licenses and Convey the resulting work. Corresponding
* source of such a combination shall include the source code for all other
* software used.
*
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* See COPYING file for full licensing terms.
* See https://www.nexedi.com/licensing for rationale and options.
*/
/*jslint nomen: true */
/*global indexedDB, RSVP, IDBOpenDBRequest, DOMError, Event, Set, DOMException,
window*/
(function (indexedDB, RSVP, IDBOpenDBRequest, DOMError, DOMException) {
"use strict";
function waitForIDBRequest(request) {
return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject;
request.onsuccess = resolve;
});
}
function waitForAllSynchronousCursor(request, callback) {
var force_cancellation = false;
function canceller() {
force_cancellation = true;
}
function resolver(resolve, reject) {
request.onerror = reject;
request.onsuccess = function (evt) {
var cursor = evt.target.result;
if (cursor && !force_cancellation) {
try {
callback(cursor);
} catch (error) {
reject(error);
}
// continue to next iteration
cursor["continue"]();
} else {
resolve();
}
};
}
return new RSVP.Promise(resolver, canceller);
}
function waitForOpenIndexedDB(db_name, version, upgrade_handler, callback) {
var request;
function canceller() {
if ((request !== undefined) && (request.result !== undefined)) {
request.result.close();
}
}
function resolver(resolve, reject) {
// Open DB //
request = indexedDB.open(db_name, version);
request.onerror = function (error) {
canceller();
if ((error !== undefined) &&
(error.target instanceof IDBOpenDBRequest) &&
((error.target.error instanceof DOMError) ||
(error.target.error instanceof DOMException))) {
reject("Connection to: " + db_name + " failed: " +
error.target.error.message);
} else {
reject(error);
}
};
request.onabort = function () {
canceller();
reject("Aborting connection to: " + db_name);
};
request.ontimeout = function () {
reject("Connection to: " + db_name + " timeout");
};
request.onblocked = function () {
canceller();
reject("Connection to: " + db_name + " was blocked");
};
// Create DB if necessary //
request.onupgradeneeded = upgrade_handler;
request.onversionchange = function () {
canceller();
reject(db_name + " was upgraded");
};
request.onsuccess = function () {
var result;
try {
result = callback(request.result);
} catch (error) {
reject(error);
}
return new RSVP.Queue()
.push(function () {
return result;
})
.push(function (final_result) {
canceller();
resolve(final_result);
}, function (error) {
canceller();
reject(error);
});
};
}
return new RSVP.Promise(resolver, canceller);
}
function waitForRepairableOpenIndexedDB(db_name, version, index_keys,
sub_storage_description, signature_storage_name, upgrade_handler,
callback) {
var handleUpgradeNeeded = function (evt) {
return upgrade_handler(evt, index_keys, sub_storage_description,
signature_storage_name);
};
return waitForOpenIndexedDB(db_name, version, handleUpgradeNeeded,
callback);
}
function waitForTransaction(db, stores, flag, callback) {
var tx = db.transaction(stores, flag);
function canceller() {
try {
tx.abort();
} catch (unused) {
// Transaction already finished
return;
}
}
function resolver(resolve, reject) {
var result;
try {
result = callback(tx);
} catch (error) {
reject(error);
}
tx.oncomplete = function () {
return new RSVP.Queue()
.push(function () {
return result;
})
.push(resolve, function (error) {
canceller();
reject(error);
});
};
tx.onerror = function (error) {
canceller();
reject(error);
};
tx.onabort = function (evt) {
reject(evt.target);
};
return tx;
}
return new RSVP.Promise(resolver, canceller);
}
window.waitForTransaction = waitForTransaction;
window.waitForOpenIndexedDB = waitForOpenIndexedDB;
window.waitForIDBRequest = waitForIDBRequest;
window.waitForAllSynchronousCursor = waitForAllSynchronousCursor;
window.waitForRepairableOpenIndexedDB = waitForRepairableOpenIndexedDB;
}(indexedDB, RSVP, IDBOpenDBRequest, DOMError, DOMException));
\ No newline at end of file
src/jio.storage/indexeddbstorage.js
View file @ 61749ec2
This diff is collapsed.
src/jio.storage/indexstorage2.js
View file @ 61749ec2
......@@ -18,11 +18,14 @@
* See https://www.nexedi.com/licensing for rationale and options.
*/
/*jslint nomen: true */
/*global indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, Event,
parseStringToObject, Set, DOMException*/
(function (indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError,
parseStringToObject, DOMException) {
/*global indexedDB, jIO, RSVP, Event,
parseStringToObject, Set, waitForTransaction,
waitForRepairableOpenIndexedDB, waitForAllSynchronousCursor,
waitForIDBRequest*/
(function (indexedDB, jIO, RSVP,
parseStringToObject, waitForTransaction, waitForRepairableOpenIndexedDB,
waitForAllSynchronousCursor, waitForIDBRequest) {
"use strict";
function IndexStorage2(description) {
......@@ -74,40 +77,6 @@
return doc;
}
function waitForIDBRequest(request) {
return new RSVP.Promise(function (resolve, reject) {
request.onerror = reject;
request.onsuccess = resolve;
});
}
function waitForAllSynchronousCursor(request, callback) {
var force_cancellation = false;
function canceller() {
force_cancellation = true;
}
function resolver(resolve, reject) {
request.onerror = reject;
request.onsuccess = function (evt) {
var cursor = evt.target.result;
if (cursor && !force_cancellation) {
try {
callback(cursor);
} catch (error) {
reject(error);
}
// continue to next iteration
cursor["continue"]();
} else {
resolve();
}
};
}
return new RSVP.Promise(resolver, canceller);
}
function getCursorResult(cursor, limit) {
var result = [], count = 0;
function pushLimitedMetadata(cursor) {
......@@ -234,8 +203,6 @@
}
}
var transaction_failure_reason;
function repairInTransaction(sub_storage_description, transaction,
index_keys, signature_storage_name, clear_storage) {
var repair_promise, repeatUntilPromiseFulfilled, store,
......@@ -269,7 +236,6 @@
}
if (repair_promise.isRejected) {
transaction.abort();
transaction_failure_reason = repair_promise.rejectedReason;
return;
}
if (repair_promise.isFulfilled) {
......@@ -279,7 +245,7 @@
store.get("inexistent"), next_continuation_resolve);
};
};
repeatUntilPromiseFulfilled(store.get("inexistent"));
return repeatUntilPromiseFulfilled(store.get("inexistent"));
}
function handleUpgradeNeeded(evt, index_keys, sub_storage_description,
......@@ -319,122 +285,15 @@
}
}
function waitForOpenIndexedDB(db_name, version, index_keys,
sub_storage_description, signature_storage_name, callback) {
function resolver(resolve, reject) {
// Open DB //
var request = indexedDB.open(db_name, version);
request.onerror = function (error) {
var error_sub_message;
if (request.result) {
request.result.close();
}
if ((error !== undefined) &&
(error.target instanceof IDBOpenDBRequest) &&
((error.target.error instanceof DOMError) ||
(error.target.error instanceof DOMException))) {
error_sub_message = error.target.error.message;
if (transaction_failure_reason) {
error_sub_message += " " + transaction_failure_reason;
transaction_failure_reason = undefined;
}
reject("Connection to: " + db_name + " failed: " + error_sub_message);
} else {
reject(error);
}
};
request.onabort = function () {
request.result.close();
reject("Aborting connection to: " + db_name);
};
request.ontimeout = function () {
request.result.close();
reject("Connection to: " + db_name + " timeout");
};
request.onblocked = function () {
request.result.close();
reject("Connection to: " + db_name + " was blocked");
};
// Create DB if necessary //
request.onupgradeneeded = function (evt) {
handleUpgradeNeeded(evt, index_keys, sub_storage_description,
signature_storage_name);
};
request.onversionchange = function () {
request.result.close();
reject(db_name + " was upgraded");
};
request.onsuccess = function () {
return new RSVP.Queue()
.push(function () {
return callback(request.result);
})
.push(function (result) {
request.result.close();
resolve(result);
}, function (error) {
request.result.close();
reject(error);
});
};
}
return new RSVP.Promise(resolver);
}
function waitForTransaction(db, stores, flag, callback) {
var tx = db.transaction(stores, flag);
function canceller() {
try {
tx.abort();
} catch (unused) {
// Transaction already finished
return;
}
}
function resolver(resolve, reject) {
var result;
try {
result = callback(tx);
} catch (error) {
reject(error);
}
tx.oncomplete = function () {
return new RSVP.Queue()
.push(function () {
return result;
})
.push(resolve, function (error) {
canceller();
reject(error);
});
};
tx.onerror = function (error) {
canceller();
reject(error);
};
tx.onabort = function (evt) {
reject(evt.target);
};
return tx;
}
return new RSVP.Promise(resolver, canceller);
}
IndexStorage2.prototype._runQuery = function (key, value, limit) {
var context = this;
return RSVP.Queue()
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForRepairableOpenIndexedDB(context._database_name,
context._version, context._index_keys,
context._sub_storage_description, context._signature_storage_name,
handleUpgradeNeeded, function (db) {
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
if (limit) {
......@@ -485,9 +344,9 @@
if (context._index_keys.length === 0) {
return;
}
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForRepairableOpenIndexedDB(context._database_name,
context._version, context._index_keys, context._sub_storage_description,
context._signature_storage_name, handleUpgradeNeeded, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store").put({
......@@ -521,9 +380,10 @@
var context = this;
return context._sub_storage.remove(id)
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForRepairableOpenIndexedDB(context._database_name,
context._version, context._index_keys,
context._sub_storage_description, context._signature_storage_name,
handleUpgradeNeeded, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
......@@ -535,9 +395,9 @@
IndexStorage2.prototype.repair = function () {
var context = this;
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForRepairableOpenIndexedDB(context._database_name,
context._version, context._index_keys, context._sub_storage_description,
context._signature_storage_name, handleUpgradeNeeded, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return repairInTransaction(context._sub_storage_description, tx,
......@@ -560,5 +420,6 @@
};
jIO.addStorage("index2", IndexStorage2);
}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject,
DOMException));
\ No newline at end of file
}(indexedDB, jIO, RSVP, parseStringToObject,
waitForTransaction, waitForRepairableOpenIndexedDB,
waitForAllSynchronousCursor, waitForIDBRequest));
\ No newline at end of file
test/jio.storage/indexstorage2.tests.js
View file @ 61749ec2
......@@ -789,36 +789,6 @@
});
});
test("Repair fails", function () {
var context = this, chrome_error, firefox_error;
chrome_error = "Connection to: jio:index2_test failed: Version change " +
"transaction was aborted in upgradeneeded event handler. " +
"Error: Capacity 'buildQuery' is not implemented on 'dummystorage3'";
firefox_error = "Connection to: jio:index2_test failed: A request was " +
"aborted, for example through a call to IDBTransaction.abort. " +
"Error: Capacity 'buildQuery' is not implemented on 'dummystorage3'";
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(1);
DummyStorage3.prototype.buildQuery = undefined;
context.jio.allDocs({query: "c: 'control'"})
.fail(function (error) {
ok(error === chrome_error || error === firefox_error);
})
.always(function () {
start();
});
});
test("Manual repair", function () {
var context = this, fake_data;
context.jio = jIO.createJIO({
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7