Commit 5b79e0e6 authored by Guillaume Royer's avatar Guillaume Royer

feat(storage): add elasticlunr storage filter on querystorage

parent 4b8bddba
...@@ -103,6 +103,7 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \ ...@@ -103,6 +103,7 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
${EXTERNALDIR}/uritemplate.js \ ${EXTERNALDIR}/uritemplate.js \
${EXTERNALDIR}/lz-string.js \ ${EXTERNALDIR}/lz-string.js \
${EXTERNALDIR}/moment.js \ ${EXTERNALDIR}/moment.js \
${EXTERNALDIR}/elasticlunr.js \
${SRCDIR}/queries/parser-begin.js \ ${SRCDIR}/queries/parser-begin.js \
${SRCDIR}/queries/build/parser.js \ ${SRCDIR}/queries/build/parser.js \
${SRCDIR}/queries/parser-end.js \ ${SRCDIR}/queries/parser-end.js \
...@@ -130,7 +131,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \ ...@@ -130,7 +131,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
${SRCDIR}/jio.storage/cryptstorage.js \ ${SRCDIR}/jio.storage/cryptstorage.js \
${SRCDIR}/jio.storage/websqlstorage.js \ ${SRCDIR}/jio.storage/websqlstorage.js \
${SRCDIR}/jio.storage/fbstorage.js \ ${SRCDIR}/jio.storage/fbstorage.js \
${SRCDIR}/jio.storage/cloudooostorage.js ${SRCDIR}/jio.storage/cloudooostorage.js \
${SRCDIR}/jio.storage/elasticlunrstorage.js
@mkdir -p $(@D) @mkdir -p $(@D)
cat $^ > $@ cat $^ > $@
......
/*jslint nomen: true */
/*global jIO, QUnit, sinon */
(function (jIO, QUnit, sinon) {
"use strict";
var test = QUnit.test,
stop = QUnit.stop,
start = QUnit.start,
ok = QUnit.ok,
deepEqual = QUnit.deepEqual,
module = QUnit.module;
/////////////////////////////////////////////////////////////////
// Custom test substorage definition
/////////////////////////////////////////////////////////////////
function Storage200() {
return this;
}
jIO.addStorage('elasticlunr200', Storage200);
function Index200() {
return this;
}
jIO.addStorage('index200', Index200);
/////////////////////////////////////////////////////////////////
// ElasticlunrStorage.buildQuery
/////////////////////////////////////////////////////////////////
module("ElasticlunrStorage.buildQuery", {
setup: function () {
var context = this;
Index200.prototype.putAttachment = function () {
return true;
};
Index200.prototype.getAttachment = function () {
return true;
};
this.jio = jIO.createJIO({
type: "elasticlunr",
index_fields: [
"title"
],
index_sub_storage: {
type: "index200"
},
sub_storage: {
type: "drivetojiomapping",
sub_storage: {
type: "dropbox",
access_token: "sample_token"
}
}
});
this.jio.__storage._resetIndex("id", [
"title"
]);
this.documents = {};
Storage200.prototype.hasCapacity = function () {
return true;
};
Storage200.prototype.buildQuery = function (options) {
return new RSVP.Queue()
.push(function () {
// capacity query
return Object.keys(context.documents).map(function (id) {
return {
id: id,
value: context.documents[id]
};
});
})
.push(function (docs) {
// capacity filter
if (options.ids) {
return docs.filter(function (doc) {
return options.ids.indexOf(doc.id) >= 0;
});
}
return docs;
});
};
this.server = sinon.fakeServer.create();
this.server.autoRespond = true;
this.server.autoRespondAfter = 5;
},
teardown: function () {
this.server.restore();
delete this.server;
}
});
/////////////////////////////////////////////////////////////////
// Dropbox tests
/////////////////////////////////////////////////////////////////
test("dropbox: search by title", function () {
var context = this,
server = this.server,
doc = {
title: "foo",
subtitle: "bar",
desc: "empty"
};
server.respondWith(
"POST",
"https://content.dropboxapi.com/2/files/upload",
[204, {
"Content-Type": "text/xml"
}, ""]
);
server.respondWith(
"POST",
"https://content.dropboxapi.com/2/files/download",
[200, {
"Content-Type": "application/json"
}, JSON.stringify(doc)]
);
stop();
RSVP.all([
context.jio.put("1", doc),
context.jio.put("2", {
title: "bar",
subtitle: "bar",
desc: "empty"
})
])
.then(function () {
return context.jio.allDocs({
query: 'title: "foo"'
});
})
.then(function (result) {
deepEqual(result, {
data: {
rows: [{
id: "1",
value: {}
}],
total_rows: 1
}
});
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit, sinon));
...@@ -30,7 +30,10 @@ See https://www.nexedi.com/licensing for rationale and options. ...@@ -30,7 +30,10 @@ See https://www.nexedi.com/licensing for rationale and options.
<link rel="stylesheet" href="../external/qunit.css" type="text/css" media="screen"/> <link rel="stylesheet" href="../external/qunit.css" type="text/css" media="screen"/>
<script src="../external/qunit.js" type="text/javascript"></script> <script src="../external/qunit.js" type="text/javascript"></script>
<script src="../external/sinon.js" type="text/javascript"></script>
<script src="scenario.js"></script> <script src="scenario.js"></script>
<script src="elasticlunr-dropbox.scenario.js"></script>
</head> </head>
<body> <body>
......
This diff is collapsed.
/*
* Copyright 2018, Nexedi SA
*
* This program is free software: you can Use, Study, Modify and Redistribute
* it under the terms of the GNU General Public License version 3, or (at your
* option) any later version, as published by the Free Software Foundation.
*
* You can also Link and Combine this program with other software covered by
* the terms of any of the Free Software licenses or any of the Open Source
* Initiative approved licenses and Convey the resulting work. Corresponding
* source of such a combination shall include the source code for all other
* software used.
*
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* See COPYING file for full licensing terms.
* See https://www.nexedi.com/licensing for rationale and options.
*/
/*jslint sloppy: true, nomen: true */
/*global jIO, RSVP, Blob, Query, elasticlunr */
(function (jIO, RSVP, Blob, Query, elasticlunr) {
'use strict';
var elasticlunrStorageKey = 'jio_elasticlunr';
function findDuplicates(array) {
var sorted = array.slice().sort(),
results = [],
i;
for (i = 0; i < sorted.length - 1; i += 1) {
if (sorted[i + 1] === sorted[i]) {
if (results.indexOf(sorted[i]) === -1) {
results.push(sorted[i]);
}
}
}
return results;
}
function initIndex(id, indexFields) {
var index = elasticlunr();
indexFields.forEach(function (field) {
index.addField(field);
});
index.setRef(id);
// do not store the documents in the index
index.saveDocument(false);
return index;
}
function loadIndex(attachmentKey, storage, id, indexFields) {
var index = null;
return storage
.getAttachment(attachmentKey, attachmentKey, {
format: 'text'
})
.push(function (data) {
index = elasticlunr.Index.load(JSON.parse(data));
}, function () {
index = initIndex(id, indexFields);
})
.push(function () {
return index;
});
}
function searchQuery(index, indexedFields, key, value) {
var config = {
boolean: "OR",
expand: true,
fields: {}
};
if (indexedFields.indexOf(key) >= 0) {
config.fields[key] = {
boost: 1,
bool: 'AND'
};
// we can only do a single-string search, so we can
// stop on the first indexed field we find
return index.search(value, config).map(function (result) {
return result.ref;
});
}
return null;
}
function recursiveIndexQuery(index, indexedFields, query) {
var ids = null,
subquery,
i,
subids;
if (query.query_list) {
for (i = query.query_list.length - 1; i >= 0; i -= 1) {
subquery = query.query_list[i];
subids = recursiveIndexQuery(index, indexedFields, subquery);
if (subids !== null) {
query.query_list.splice(i, 1);
if (ids === null) {
ids = subids;
} else {
ids = findDuplicates(ids.concat(subids));
}
}
}
return ids;
}
return searchQuery(index, indexedFields, query.key, query.value);
}
/**
* The jIO Elasticlunr extension
*
* @class ElasticlunrStorage
* @constructor
*/
function ElasticlunrStorage(spec) {
if (!spec.index_sub_storage) {
throw new TypeError(
"Elasticlunr 'index_sub_storage' must be provided."
);
}
this._index_sub_storage = jIO.createJIO(spec.index_sub_storage);
if (!this._index_sub_storage.hasCapacity('getAttachment')) {
throw new TypeError(
"Elasticlunr 'index_sub_storage' must have getAttachment capacity."
);
}
if (!spec.sub_storage) {
throw new TypeError(
"Elasticlunr 'sub_storage' must be provided."
);
}
this._sub_storage = jIO.createJIO(spec.sub_storage);
this._index_sub_storage_key = elasticlunrStorageKey + '_' +
this._sub_storage.__type;
this._index_id = spec.id || 'id';
this._index_fields = spec.index_fields || [];
}
ElasticlunrStorage.prototype._getIndex = function () {
var context = this;
if (this._index) {
return new RSVP.Queue().push(function () {
return context._index;
});
}
return loadIndex(
this._index_sub_storage_key,
this._index_sub_storage,
this._index_id,
this._index_fields
).push(function (index) {
context._index = index;
return context._index;
});
};
ElasticlunrStorage.prototype._resetIndex = function (id, indexFields) {
this._index_id = 'id';
this._index_fields = indexFields;
this._index = initIndex(id, indexFields);
};
ElasticlunrStorage.prototype._saveIndex = function () {
var context = this;
return this._getIndex()
.push(function (index) {
var data = JSON.stringify(index);
return context._index_sub_storage.putAttachment(
context._index_sub_storage_key,
context._index_sub_storage_key,
new Blob([data])
);
});
};
ElasticlunrStorage.prototype.get = function () {
return this._sub_storage.get.apply(this._sub_storage, arguments);
};
ElasticlunrStorage.prototype.allAttachments = function () {
return this._sub_storage.allAttachments.apply(
this._sub_storage,
arguments
);
};
ElasticlunrStorage.prototype.post = function (doc) {
var context = this;
return this._sub_storage.post.apply(this._sub_storage, arguments)
.push(function (id) {
var data = JSON.parse(JSON.stringify(doc));
data.id = id.toString();
return context._getIndex().push(function (index) {
index.addDoc(data);
return context._saveIndex();
});
});
};
ElasticlunrStorage.prototype.put = function (id, doc) {
var context = this;
return this._sub_storage.put.apply(this._sub_storage, arguments)
.push(function () {
var data = JSON.parse(JSON.stringify(doc));
data.id = id.toString();
return context._getIndex().push(function (index) {
index.updateDoc(data);
return context._saveIndex();
});
});
};
ElasticlunrStorage.prototype.remove = function (id) {
var context = this;
// need to get the full document to remove every data from indexes
return this._sub_storage.get(id)
.push(function (doc) {
return context._sub_storage.remove(id)
.push(function () {
return context._getIndex();
})
.push(function (index) {
index.removeDoc(doc);
return context._saveIndex();
});
});
};
ElasticlunrStorage.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(
this._sub_storage,
arguments
);
};
ElasticlunrStorage.prototype.putAttachment = function () {
return this._sub_storage.putAttachment.apply(
this._sub_storage,
arguments
);
};
ElasticlunrStorage.prototype.removeAttachment = function () {
return this._sub_storage.removeAttachment.apply(
this._sub_storage,
arguments
);
};
ElasticlunrStorage.prototype.repair = function () {
// rebuild index?
return this._sub_storage.repair.apply(this._sub_storage, arguments);
};
ElasticlunrStorage.prototype.hasCapacity = function (name) {
var capacityList = [
'limit', 'sort', 'select', 'query'
];
if (capacityList.indexOf(name) !== -1) {
return true;
}
if (name === 'index') {
return true;
}
return this._sub_storage.hasCapacity(name);
};
ElasticlunrStorage.prototype.buildQuery = function (options) {
var context = this,
indexedFields = this._index_fields,
runSubstorageQuery = options.select_list || options.include_docs,
parsedQuery;
if (options.query && options.query.indexOf('OR') === -1) {
parsedQuery = jIO.QueryFactory.create(options.query);
return context._getIndex()
.push(function (index) {
return recursiveIndexQuery(index, indexedFields, parsedQuery);
})
.push(function (ids) {
try {
if (context._sub_storage.hasCapacity('query_filtered')) {
// simple query with found matches, just exec a simple list
if ((ids || []).length && parsedQuery.type === 'simple') {
delete options.query;
} else {
options.query = Query.objectToSearchText(parsedQuery);
}
options.ids = ids;
return context._sub_storage.buildQuery(options);
}
} catch (ignore) {}
// run query with substorage if we want to retrieve the documents
if (runSubstorageQuery) {
return context._sub_storage.buildQuery(options);
}
return (ids || []).map(function (id) {
return {
id: id,
value: {}
};
});
});
}
return this._sub_storage.buildQuery(options);
};
jIO.addStorage('elasticlunr', ElasticlunrStorage);
}(jIO, RSVP, Blob, Query, elasticlunr));
This diff is collapsed.
...@@ -76,6 +76,7 @@ See https://www.nexedi.com/licensing for rationale and options. ...@@ -76,6 +76,7 @@ See https://www.nexedi.com/licensing for rationale and options.
<script src="jio.storage/websqlstorage.tests.js"></script> <script src="jio.storage/websqlstorage.tests.js"></script>
<script src="jio.storage/fbstorage.tests.js"></script> <script src="jio.storage/fbstorage.tests.js"></script>
<script src="jio.storage/httpstorage.tests.js"></script> <script src="jio.storage/httpstorage.tests.js"></script>
<script src="jio.storage/elasticlunrstorage.tests.js"></script>
<!--script src="../src/jio.storage/xwikistorage.js"></script> <!--script src="../src/jio.storage/xwikistorage.js"></script>
<script src="jio.storage/xwikistorage.tests.js"></script--> <script src="jio.storage/xwikistorage.tests.js"></script-->
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment