Skip to content

jio
jio
Commit 4a6e56c2 authored by Tristan Cavelier's avatar Tristan Cavelier
Browse files
Options

Merge branch 'queries'

parents 7ca3f513 ffc30121
Hide whitespace changes
Inline Side-by-side
Showing with 2278 additions and 1626 deletions
Makefile
View file @ 4a6e56c2
......@@ -11,6 +11,8 @@ COMPLEX_MIN = complex_queries.min.js
PARSER_PAR = $(QUERIES_DIR)/parser.par
PARSER_OUT = $(QUERIES_DIR)/parser.js
## install npm package system wide -> npm -g install <package>
## js/cc using rhino
#JSCC_CMD = rhino ~/modules/jscc/jscc.js -t ~/modules/jscc/driver_web.js_
# sh -c 'cd ; npm install jscc-node'
......@@ -19,6 +21,8 @@ JSCC_CMD = node ~/node_modules/jscc-node/jscc.js -t ~/node_modules/jscc-node/
LINT_CMD = $(shell which jslint || echo node ~/node_modules/jslint/bin/jslint.js) --terse
# sh -c 'cd ; npm install uglify-js'
UGLIFY_CMD = $(shell which uglifyjs || echo node ~/node_modules/uglify-js/bin/uglifyjs)
# sh -c 'cd ; npm install phantomjs'
PHANTOM_CMD = $(shell which phantomjs || echo ~/node_modules/phantomjs/bin/phantomjs)
auto: compile build lint
build: concat uglify
......@@ -26,7 +30,7 @@ build: concat uglify
# The order is important!
CONCAT_JIO_NAMES = intro exceptions jio.intro storages/* commands/* jobs/status/* jobs/job announcements/announcement activityUpdater announcements/announcer jobs/jobIdHandler jobs/jobManager jobs/jobRules jio.core jio.outro jioNamespace outro
CONCAT_STORAGE_NAMES = *
CONCAT_QUERIES_NAMES = begin parser-begin parser parser-end serializer query end
CONCAT_QUERIES_NAMES = begin parser-begin parser parser-end tool queryfactory query simplequery complexquery end
LINT_NAMES = exceptions storages/* commands/* jobs/status/* jobs/* announcements/* activityUpdater jio.core jioNamespace
CONCAT_QUERIES_FILES = $(CONCAT_QUERIES_NAMES:%=$(QUERIES_DIR)/%.js)
......@@ -57,7 +61,7 @@ lint:
$(LINT_CMD) $(LINT_FILES)
phantom:
~/node_modules/phantomjs/bin/phantomjs test/run-qunit.js test/jiotests_withoutrequirejs.html | awk 'BEGIN {print "<!DOCTYPE html><html>"} /^<head>$$/, /^<\/body>$$/ {print} END {print "</html>"}' | sed -e 's,^ *<\(/\|\)script.*>$$,,g' > test/unit_test_result.html
$(PHANTOM_CMD) test/run-qunit.js test/jiotests_withoutrequirejs.html | awk 'BEGIN {print "<!DOCTYPE html><html>"} /^<head>$$/, /^<\/body>$$/ {print} END {print "</html>"}' | sed -e 's,^ *<\(/\|\)script.*>$$,,g' > test/unit_test_result.html
grep '^ <title>✔ ' test/unit_test_result.html > /dev/null
.phony: clean
......
examples/example-queries.html
View file @ 4a6e56c2
......@@ -16,11 +16,11 @@
<table>
<tr>
<td>Query (String):<br /><textarea id="str">1:abc AND 2:def</textarea></td>
<td>Query (Object):<br /><textarea id="obj">{&quot;type&quot;:&quot;complex&quot;,&quot;operator&quot;:&quot;AND&quot;,&quot;query_list&quot;:[{&quot;type&quot;:&quot;simple&quot;,&quot;operator&quot;:&quot;=&quot;,&quot;id&quot;:&quot;1&quot;,&quot;value&quot;:&quot;abc&quot;},{&quot;type&quot;:&quot;simple&quot;,&quot;operator&quot;:&quot;=&quot;,&quot;id&quot;:&quot;2&quot;,&quot;value&quot;:&quot;def&quot;}]}</textarea></td>
<td>Query (Object):<br /><textarea id="obj">{&quot;type&quot;:&quot;complex&quot;,&quot;operator&quot;:&quot;AND&quot;,&quot;query_list&quot;:[{&quot;type&quot;:&quot;simple&quot;,&quot;operator&quot;:&quot;=&quot;,&quot;key&quot;:&quot;1&quot;,&quot;value&quot;:&quot;abc&quot;},{&quot;type&quot;:&quot;simple&quot;,&quot;operator&quot;:&quot;=&quot;,&quot;key&quot;:&quot;2&quot;,&quot;value&quot;:&quot;def&quot;}]}</textarea></td>
</tr>
<tr>
<td>Object List:<br /><textarea id="list">[{&quot;1&quot;:&quot;abc&quot;,&quot;2&quot;:&quot;def&quot;},{&quot;1&quot;:&quot;def&quot;,&quot;2&quot;:&quot;abc&quot;}]</textarea></td>
<td>Result (Query String):<br /><textarea id="result">[{&quot;1&quot;:&quot;abc&quot;,&quot;2&quot;:&quot;def&quot;}]</textarea></td>
<td>Item list (to filter, from 'Query (Object)'):<br /><textarea id="list">[{&quot;1&quot;:&quot;abc&quot;,&quot;2&quot;:&quot;def&quot;},{&quot;1&quot;:&quot;def&quot;,&quot;2&quot;:&quot;abc&quot;}]</textarea></td>
<td>Result list:<br /><textarea id="result">[{&quot;1&quot;:&quot;abc&quot;,&quot;2&quot;:&quot;def&quot;}]</textarea></td>
</tr>
<tr>
<td><label for="wildcard">Wildcard char: </label></td>
......@@ -39,38 +39,36 @@
<td><input type="text" id="limit" name="limit" value="[0,100]" /></td>
</tr>
</table>
<button onclick="parse()">Parse</button>
<button onclick="serialize()">Serialize</button>
<button onclick="searchTextToJson()">Search text to JSON</button>
<button onclick="jsonToSearchText()">JSON to Search text</button>
<button onclick="query()">Query</button>
<script type="text/javascript" src="../lib/md5/md5.js"></script>
<script type="text/javascript" src="../lib/jsSha2/sha2.js"></script>
<script type="text/javascript" src="../lib/sjcl/sjcl.min.js"></script>
<script type="text/javascript" src="../jio.min.js"></script>
<script type="text/javascript" src="../complex_queries.min.js"></script>
<script type="text/javascript"
<script type="text/javascript"
src="http://code.jquery.com/jquery-1.8.2.min.js"></script>
<script type="text/javascript">
<!--
var parse = function () {
$('#obj').attr('value',JSON.stringify(jIO.ComplexQueries.parse($('#str').attr('value'))));
var searchTextToJson = function () {
$("#obj").attr("value", JSON.stringify(complex_queries.Query.parseStringToObject($("#str").attr("value"))));
};
var serialize = function () {
$('#str').attr('value',jIO.ComplexQueries.serialize(JSON.parse($('#obj').attr('value'))));
var jsonToSearchText = function () {
$("#str").attr("value", complex_queries.QueryFactory.create(JSON.parse($("#obj").attr("value"))).toString());
};
var query = function () {
$('#result').attr('value',JSON.stringify(
jIO.ComplexQueries.query(
{
query:$('#str').attr('value'),
filter:{
sort_on:JSON.parse($('#sort_on').attr('value')),
limit:JSON.parse($('#limit').attr('value')),
select_list:JSON.parse($('#select_list').attr('value'))
},
wildcard_character:$('#wildcard').attr('value')
}, JSON.parse($('#list').attr('value'))
)
));
var list = JSON.parse($("#list").attr("value"));
$("#str").attr("value", complex_queries.QueryFactory.create(JSON.parse($("#obj").attr("value"))).exec(
list,
{
"wildcard_character": $('#wildcard').attr('value'),
"sort_on": JSON.parse($("#sort_on").attr("value")),
"limit": JSON.parse($("#limit").attr("value")),
"select_list": JSON.parse($("#select_list").attr("value"))
}
));
$("#result").attr("value", JSON.stringify(list));
};
// -->
</script>
......
examples/jio_dashboard.html
View file @ 4a6e56c2
......@@ -92,8 +92,6 @@ var clearlog = function () {
</td>
<td colspan="1" style="text-align: center;">
Options:<br />
<label for="include_docs">Include Docs</label>
<input type="checkbox" id="include_docs" /><br />
<label for="show_conflicts">Get Conflicts</label>
<input type="checkbox" id="show_conflicts" /><br />
<label for="show_revision_history">Get Revision History</label>
......@@ -101,7 +99,7 @@ var clearlog = function () {
<label for="show_revision_info">Get Revision Info</label>
<input type="checkbox" id="show_revision_info" /><br />
<label for="max_retry">Max Retry</label>
<input type="number" id="max_retry" value="0" style="width:30px;"/>
<input type="number" id="max_retry" value="0" style="width: 3em;"/>
(0 = infinite)
</td>
</tr>
......@@ -111,14 +109,36 @@ var clearlog = function () {
<button onclick="put()">put</button>
<button onclick="get()">get</button>
<button onclick="remove()">remove</button>
<button onclick="allDocs()">allDocs</button>
<!-- </td> -->
<!-- <td style="text-align: center;"> -->
- <button onclick="putAttachment()">putAttachment</button>
<button onclick="getAttachment()">getAttachment</button>
<button onclick="removeAttachment()">removeAttachment</button>
</td>
</tr>
<tr>
<td colspan="1" style="width: 50%;">
<label for="query">AllDocs Query:</label>
<textarea id="query" rows="3" style="width: 98%;">a: 2</textarea>
</td>
<td colspan="1" style="text-align: center;">
AllDocs Options:<br />
<label for="include_docs">Include Docs</label>
<input type="checkbox" id="include_docs" /><br />
<label for="wildcard">Wildcard char: </label>
<input type="text" id="wildcard" name="wildcard" value="%" maxlength="1" style="width: 1em;"/><br />
<label for="sort_on">Sort on: </label>
<input type="text" id="sort_on" name="sort_on" value="[[&quot;author&quot;, &quot;ascending&quot;], [&quot;title&quot;, &quot;descending&quot;]]"
style="width: 80%;"/><br />
<label for="select_list">Select_list: </label>
<input type="text" id="select_list" name="select_list" value="[&quot;author&quot;, &quot;title&quot;]" style="width: 80%;"/><br />
<label for="limit">Limit: </label>
<input type="text" id="limit" name="limit" value="[0, 100]" style="width: 80%;"/><br />
</td>
</tr>
<tr>
<td colspan="2" style="text-align: center;">
<button onclick="allDocs()">allDocs</button>
</td>
</tr>
</table>
<br />
<div style="text-align: center;">
......@@ -130,6 +150,7 @@ var clearlog = function () {
<div id="log">
</div>
<script type="text/javascript" src="../lib/md5/md5.js"></script>
<script type="text/javascript" src="../complex_queries.js"></script>
<script type="text/javascript" src="../jio.js"></script>
<script type="text/javascript" src="../src/jio.storage/localstorage.js">
</script>
......@@ -223,7 +244,13 @@ var command = function (method) {
return error('no jio set');
}
opts.query = $('#query').attr('value');
opts.include_docs = $('#include_docs').attr('checked') ? true : false;
opts.wildcard_character = $('#wildcard').attr('value') || "";
opts.sort_on = JSON.parse($('#sort_on').attr('value') || null);
opts.select_list = JSON.parse($('#select_list').attr('value') || null);
opts.limit = JSON.parse($('#limit').attr('value') || null);
opts.conflicts = $('#show_conflicts').attr('checked') ? true : false;
opts.revs = $('#show_revision_history').attr('checked') ? true : false;
opts.revs_info = $('#show_revision_info').attr('checked') ? true : false;
......
grunt/15_gruntQueries/Makefile 0 → 100644
View file @ 4a6e56c2
JSCCDIR = /opt/jscc
JSCCCMD = rhino $(JSCCDIR)/jscc.js -t $(JSCCDIR)/driver_web.js_
SRC_DIR = ../../src/queries
BUILD_DIR = ../../built/queries
OUT = $(BUILD_DIR)/complex-queries.js
PARSER_PAR = $(SRC_DIR)/parser.par
PARSER_OUT = $(BUILD_DIR)/parser.js
auto: prepare parser grunt
prepare:
mkdir -p $(BUILD_DIR)
parser:
$(JSCCCMD) $(PARSER_PAR) >> $(PARSER_OUT)
grunt:
grunt
grunt/15_gruntQueries/grunt.js 0 → 100644
View file @ 4a6e56c2
/*global module:false*/
module.exports = function(grunt) {
// Project configuration.
grunt.initConfig({
pkg: '<json:package.json>',
meta: {
banner: '/*! <%= pkg.title || pkg.name %> - v<%= pkg.version %> - '+
'<%= grunt.template.today("yyyy-mm-dd") %>\n' +
'<%= pkg.homepage ? "* " + pkg.homepage + "\n" : "" %>' +
'* Copyright (c) <%= grunt.template.today("yyyy") %> Nexedi;' +
' Licensed <%= _.pluck(pkg.licenses, "type").join(", ") %> */'
},
concat: {
dist: {
src: ['<banner:meta.banner>',
// Wrapper top
'<file_strip_banner:../../src/queries/begin.js>',
// code
'<file_strip_banner:../../src/queries/parser-begin.js>',
'<file_strip_banner:../../built/queries/parser.js>',
'<file_strip_banner:../../src/queries/parser-end.js>',
'<file_strip_banner:../../src/queries/serializer.js>',
'<file_strip_banner:../../src/queries/query.js>',
// Wrapper bottom
'<file_strip_banner:../../src/queries/end.js>'],
dest: '../../<%= pkg.name %>.js'
}
},
min: {
dist: {
src: ['<banner:meta.banner>', '<config:concat.dist.dest>'],
dest: '../../<%= pkg.name %>.min.js'
}
},
qunit: {
files: ['../../test/cq-tests.html']
},
lint: {
files: ['grunt.js',
'../../src/queries/serializer.js',
'../../src/queries/query.js']
},
watch: {
files: '<config:lint.files>',
tasks: 'lint qunit'
},
jshint: {
options: {
curly: true,
eqeqeq: true,
immed: true,
latedef: true,
newcap: true,
noarg: true,
sub: true,
undef: true,
boss: true,
eqnull: true,
browser: true
},
globals: {
scope: true,
console: true,
unescape: true,
// Needed to avoid "not defined error" with requireJs
define: true,
require: true,
// Needed to avoid "not defined error" with sinonJs
sinon: true,
module: true,
test: true,
ok: true,
deepEqual: true,
expect: true,
stop: true,
start: true,
equal: true
}
},
uglify: {}
});
// Default task.
grunt.registerTask('default', 'lint concat min');
};
grunt/15_gruntQueries/package.json 0 → 100644
View file @ 4a6e56c2
{
"name": "complex-queries",
"title": "Complex Queries",
"description": "Complex Queries",
"version": "0.1.0",
"homepage": "",
"author": {
"name": "Tristan Cavelier",
"email": "tristan.cavelier@tiolive.com"
},
"repository": {
"type": "git",
"url": "http://git.erp5.org/repos/jio.git"
},
"bugs": {
"url": ""
},
"licenses": [
],
"dependencies": {
},
"keywords": []
}
src/jio.storage/erp5storage.js
View file @ 4a6e56c2
......@@ -301,9 +301,16 @@ jIO.addStorageType("erp5", function (spec, my) {
* @param {string} method The ERP5 request method
*/
priv.genericCommand = function (command, method) {
var option = command.cloneOption();
if (complex_queries !== undefined &&
method === 'allDocs' &&
option.query) {
option.query =
complex_queries.QueryFactory.create(option.query || "").serialized();
}
erp5.genericRequest(
command.cloneDoc(),
command.cloneOption(),
option,
method
).always(function (err, response) {
if (err) {
......
src/jio.storage/indexstorage.js
View file @ 4a6e56c2
/*
* Copyright 2013, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global jIO: true, localStorage: true, setTimeout: true */
* Copyright 2013, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true, regexp: true */
/*global jIO: true, localStorage: true, define: true, complex_queries: true */
/**
* JIO Index Storage.
* Manages indexes for specified storages.
* Description:
* {
* "type": "index",
* "indices": [
* {"indexA",["field_A"]},
* {"indexAB",["field_A","field_B"]}
* ],
* "field_types": {
* "field_A": "dateTime",
* "field_B": "string"
* },
* "storage": [
* <sub storage description>,
* ...
* ]
* "type": "index",
* "indices": [{
* "id": "index_title_subject.json", // doc id where to store indices
* "index": ["title", "subject"] // metadata to index
* "sub_storage": <sub storage where to store index>
* (default equal to parent sub_storage field)
* }, {
* "id": "index_year.json",
* "index": "year"
* ...
* }],
* "sub_storage": <sub storage description>
* }
* Index file will contain
*
* Sent document metadata will be:
* index_titre_subject.json
* {
* "_id": "app-name_indices.json",
* "indexA":
* "fieldA": {
* "keyword_abc": ["some_id","some_other_id",...]
* }
* "_id": "index_title_subject.json",
* "indexing": ["title", "subject"],
* "free": [0],
* "location": {
* "foo": 1,
* "bar": 2,
* ...
* },
* "indexAB": {
* "fieldA": {
* "keyword_abc": ["some_id"]
* },
* "fieldB": {
* "keyword_def": ["some_id"]
* }
* }
* "database": [
* {},
* {"_id": "foo", "title": "...", "subject": ...},
* {"_id": "bar", "title": "...", "subject": ...},
* ...
* ]
* }
* NOTES:
* It may be difficult to "un-sort" multi-field indices, like
* indexAB, because all keywords will be listed regrardless
* of underlying field, so an index on author and year would produce
* two entries per record like:
*
* "William Shakespeare":["id_Romeo_and_Juliet", "id_Othello"],
* "1591":["id_Romeo_and_Juliet"],
* "1603":["id_Othello"]
*
* So for direct lookups, this should be convient, but for other types
* of queries, it depends
*
* index_year.json
* {
* "_id": "index_year.json",
* "indexing": ["year"],
* "free": [1],
* "location": {
* "foo": 0,
* "bar": 2,
* ...
* },
* "database": [
* {"_id": "foo", "year": "..."},
* {},
* {"_id": "bar", "year": "..."},
* ...
* ]
* }
*
* A put document will be indexed to the free location if exist, else it will be
* indexed at the end of the database. The document id will be indexed, also, in
* 'location' to quickly replace metadata.
*
* Only one or two loops are executed:
* - one to filter retrieved document list (no query -> no loop)
* - one to format the result to a JIO response
*/
jIO.addStorageType('indexed', function (spec, my) {
(function () {
"use strict";
var that, priv = {};
spec = spec || {};
that = my.basicStorage(spec, my);
priv.indices = spec.indices;
priv.field_types = spec.field_types;
priv.substorage_key = "sub_storage";
priv.substorage = spec[priv.substorage_key];
priv.index_indicator = spec.sub_storage.application_name || "index";
priv.index_suffix = priv.index_indicator + "_indices.json";
var error_dict = {
"Corrupted Index": {
"status": 24,
"statusText": "Corrupt",
"error": "corrupt",
"reason": "corrupted index database"
},
"Corrupted Metadata": {
"status": 24,
"statusText": "Corrupt",
"error": "corrupt",
"reason": "corrupted document"
},
"Not Found": {
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"reason": "missing document"
},
"Conflict": {
"status": 409,
"statusText": "Conflicts",
"error": "conflicts",
"reason": "already exist"
},
"Different Index": {
"status": 40,
"statusText": "Check failed",
"error": "check_failed",
"reason": "incomplete database"
}
};
my.env = my.env || spec.env || {};
/**
* Generate a JIO Error Object
*
* @method generateErrorObject
* @param {String} name The error name
* @param {String} message The error message
* @param {String} [reason] The error reason
* @return {Object} A jIO error object
*/
function generateErrorObject(name, message, reason) {
if (!error_dict[name]) {
return {
"status": 0,
"statusText": "Unknown",
"error": "unknown",
"message": message,
"reason": reason || "unknown"
};
}
return {
"status": error_dict[name].status,
"statusText": error_dict[name].statusText,
"error": error_dict[name].error,
"message": message,
"reason": reason || error_dict[name].reason
};
}
that.specToStore = function () {
var o = {};
o[priv.substorage_key] = priv.substorage;
o.env = my.env;
return o;
};
/**
* Get the real type of an object
* @method type
* @param {Any} value The value to check
* @return {String} The value type
*/
function type(value) {
// returns "String", "Object", "Array", "RegExp", ...
return (/^\[object ([a-zA-Z]+)\]$/).exec(
Object.prototype.toString.call(value)
)[1];
}
/**
* Generate a new uuid
* @method generateUuid
* @return {string} The new uuid
*/
priv.generateUuid = function () {
function generateUuid() {
var S4 = function () {
var i, string = Math.floor(
Math.random() * 0x10000 /* 65536 */
......@@ -98,863 +168,687 @@ jIO.addStorageType('indexed', function (spec, my) {
S4() + "-" +
S4() + "-" +
S4() + S4() + S4();
};
/**
* Get number of elements in object
* @method getObjectSize
* @param {object} obj The object to check
* @return {number} size The amount of elements in the object
*/
priv.getObjectSize = function (obj) {
var size = 0, key;
for (key in obj) {
if (obj.hasOwnProperty(key)) {
size += 1;
}
}
return size;
};
}
/**
* Creates an empty indices array
* @method createEmptyIndexArray
* @param {array} indices An array of indices (optional)
* @return {object} The new index array
* A JSON Index manipulator
*
* @class JSONIndex
* @constructor
*/
priv.createEmptyIndexArray = function (indices) {
var i, k, j = priv.indices.length, new_index,
new_index_object = {}, new_index_name, new_index_fields;
if (indices === undefined) {
for (i = 0; i < j; i += 1) {
new_index = priv.indices[i];
new_index_name = new_index.name;
new_index_fields = new_index.fields;
new_index_object[new_index_name] = {};
// loop index fields and add objects to hold value/id pairs
for (k = 0; k < new_index_fields.length; k += 1) {
new_index_object[new_index_name][new_index_fields[k]] = {};
}
function JSONIndex(spec) {
var that = this;
spec = spec || {};
/**
* The document id
*
* @property _id
* @type String
*/
that._id = spec._id;
/**
* The array with metadata key to index
*
* @property _indexing
* @type Array
*/
that._indexing = spec.indexing || [];
/**
* The array of free location index
*
* @property _free
* @type Array
* @default []
*/
that._free = spec.free || [];
/**
* The dictionnary document id -> database index
*
* @property _location
* @type Object
* @default {}
*/
that._location = spec.location || {};
/**
* The database array containing document metadata
*
* @property _database
* @type Array
* @default []
*/
that._database = spec.database || [];
/**
* Adds a metadata object in the database, replace if already exist
*
* @method put
* @param {Object} meta The metadata to add
* @return {Boolean} true if added, false otherwise
*/
that.put = function (meta) {
var underscored_meta_re = /^_.*$/, k, needed_meta = {}, ok = false;
if (typeof meta._id !== "string" && meta._id !== "") {
throw new TypeError("Corrupted Metadata");
}
}
return new_index_object;
};
/**
* Determine if a key/value pair exists in an object by VALUE
* @method searchObjectByValue
* @param {object} indexToSearch The index to search
* @param {string} docid The document id to find
* @param {string} passback The value that should be returned
* @return {boolean} true/false
*/
priv.searchIndexByValue = function (indexToSearch, docid, passback) {
var key, obj, prop;
for (key in indexToSearch) {
if (indexToSearch.hasOwnProperty(key)) {
obj = indexToSearch[key];
for (prop in obj) {
if (obj[prop] === docid) {
return passback === "bool" ? true : key;
for (k in meta) {
if (meta.hasOwnProperty(k)) {
if (underscored_meta_re.test(k)) {
if (k === "_id") {
needed_meta[k] = meta[k];
}
} else if (that._indexing_object[k]) {
needed_meta[k] = meta[k];
ok = true;
}
}
}
}
return false;
};
/**
* Get element position in array
* @method getPositionInArray
* @param {object} indices The index file
* @param {object} indices The index file
* @returns {number} i Position of element in array
*/
priv.getPositionInArray = function (element, array) {
var i, l = array.length;
for (i = 0; i < l; i += 1) {
if (array[i] === element) {
return i;
if (ok) {
if (typeof that._location[meta._id] === "number") {
that._database[that._location[meta._id]] = needed_meta;
} else if (that._free.length > 0) {
k = that._free.shift();
that._database[k] = needed_meta;
that._location[meta._id] = k;
} else {
that._database.push(needed_meta);
that._location[meta._id] = that._database.length - 1;
}
return true;
}
}
return null;
};
if (typeof that._location[meta._id] === "number") {
that.remove(meta);
}
return false;
};
/**
* Find id in indices
* @method isDocidInIndex
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
* @return {boolean} true/false
*/
priv.isDocidInIndex = function (indices, doc) {
var index, i, j, label, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
index.current_size = priv.getObjectSize(index.current[label]);
// check for existing entries to remove (put-update)
if (index.current_size > 0) {
if (priv.searchIndexByValue(index.current[label], doc._id, "bool")) {
return true;
}
}
/**
* Removes a metadata object from the database if exist
*
* @method remove
* @param {Object} meta The metadata to remove
*/
that.remove = function (meta) {
if (typeof meta._id !== "string") {
throw new TypeError("Corrupted Metadata");
}
}
return false;
};
if (typeof that._location[meta._id] !== "number") {
throw new ReferenceError("Not Found");
}
that._database[that._location[meta._id]] = null;
that._free.push(that._location[meta._id]);
delete that._location[meta._id];
};
/**
* Clean up indexes when removing a file
* @method cleanIndices
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
* @return {object} indices The cleaned up file
*/
priv.cleanIndices = function (indices, doc) {
var i, j, k, index, key, label, l = priv.indices.length;
// loop indices (indexA, indexAB...)
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
// loop index fields
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
index.current_size = priv.getObjectSize(index.current[label]);
// loop field entries
for (k = 0; k < index.current_size; k += 1) {
key = priv.searchIndexByValue(index.current[label], doc._id, "key");
index.result_array = index.current[label][key];
if (!!key) {
// if there is more than one docid in the result array,
// just remove this one and not the whole array
if (index.result_array.length > 1) {
index.result_array.splice(k, 1);
} else {
delete index.current[label][key];
}
}
}
/**
* Checks if the index database document is correct
*
* @method check
*/
that.check = function () {
var id, database_meta;
if (typeof that._id !== "string" ||
that._id === "" ||
type(that._free) !== "Array" ||
type(that._indexing) !== "Array" ||
type(that._location) !== "Object" ||
type(that._database) !== "Array" ||
that._indexing.length === 0) {
throw new TypeError("Corrupted Index");
}
}
return indices;
};
/**
* Adds entries to indices
* @method createEmptyIndexArray
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
*/
priv.updateIndices = function (indices, doc) {
var i, j, index, value, label, key, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
// build array of values to create entries in index
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
value = doc[label];
if (value !== undefined) {
index.current_size = priv.getObjectSize(index.current[label]);
// check for existing entries to remove (put-update)
if (index.current_size > 0) {
key = priv.searchIndexByValue(
index.current[label],
doc._id,
"key"
);
if (!!key) {
delete index.current[label][key];
}
}
if (index.current[label][value] === undefined) {
index.current[label][value] = [];
for (id in that._location) {
if (that._location.hasOwnProperty(id)) {
database_meta = that._database[that._location[id]];
if (type(database_meta) !== "Object" ||
database_meta._id !== id) {
throw new TypeError("Corrupted Index");
}
// add a new entry
index.current[label][value].push(doc._id);
}
}
}
return indices;
};
/**
* Check available indices to find the best one.
* TODOS: NOT NICE, redo
* @method findBestIndexForQuery
* @param {object} syntax of query
* @returns {object} response The query object constructed from Index file
*/
priv.findBestIndexForQuery = function (syntax) {
var i, j, k, l, n, p, o, element, key, block,
search_ids, use_index = [], select_ids = {}, index, query_param,
current_query, current_query_size;
// try to parse into object
if (syntax.query !== undefined) {
current_query = jIO.ComplexQueries.parse(syntax.query);
} else {
current_query = {};
current_query_size = 0;
}
};
// loop indices
for (i = 0; i < priv.indices.length; i += 1) {
search_ids = [];
block = false;
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
if (current_query_size !== 0) {
// rebuild search_ids for iteration
if (current_query.query_list === undefined) {
search_ids.push(current_query.id);
} else {
for (j = 0; j < current_query.query_list.length; j += 1) {
if (priv.getPositionInArray(current_query.query_list[j].id,
search_ids) === null) {
search_ids.push(current_query.query_list[j].id);
that.equals = function (json_index) {
function equalsDirection(a, b) {
var k;
for (k in a._location) {
if (a._location.hasOwnProperty(k)) {
if (b._location[k] === undefined ||
JSON.stringify(b._database[b._location[k]]) !==
JSON.stringify(a._database[a._location[k]])) {
return false;
}
}
}
return true;
}
if (!equalsDirection(that, json_index)) {
return false;
}
if (!equalsDirection(json_index, that)) {
return false;
}
return true;
};
// loop search ids and find matches in index
for (k = 0; k < search_ids.length; k += 1) {
query_param = search_ids[0];
for (l = 0; l < index.reference_size; l += 1) {
if (query_param === index.reference.fields[l]) {
search_ids.splice(
priv.getPositionInArray(query_param, search_ids),
1
);
}
}
that.checkDocument = function (doc) {
var i, key, db_doc;
if (typeof that._location[doc._id] !== "number" ||
(db_doc = that._database(that._location[doc._id])._id) !== doc._id) {
throw new TypeError("Different Index");
}
for (i = 0; i < that._indexing.length; i += 1) {
key = that._indexing[i];
if (doc[key] !== db_doc[key]) {
throw new TypeError("Different Index");
}
}
};
// rebuild select_ids
for (o = 0; o < syntax.filter.select_list.length; o += 1) {
element = syntax.filter.select_list[o];
select_ids[element] = true;
/**
* Recreates database indices and remove free space
*
* @method repair
*/
that.repair = function () {
var i = 0, meta;
that._free = [];
that._location = {};
if (type(that._database) !== "Array") {
that._database = [];
}
// search_ids empty = all needed search fields found on index
if (search_ids.length === 0) {
p = priv.getObjectSize(select_ids);
if (p === 0) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": false
});
while (i < that._database.length) {
meta = that._database[i];
if (type(meta) === "Object" &&
typeof meta._id === "string" && meta._id !== "" &&
!that._location[meta._id]) {
that._location[meta._id] = i;
i += 1;
} else {
for (n = 0; n < index.reference_size; n += 1) {
delete select_ids[index.reference.fields[n]];
}
for (key in select_ids) {
if (select_ids.hasOwnProperty(key)) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": false
});
block = true;
}
}
if (block === false) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": true
});
}
that._database.splice(i, 1);
}
}
}
return use_index;
};
};
/**
* Returns the serialized version of this object (not cloned)
*
* @method serialized
* @return {Object} The serialized version
*/
that.serialized = function () {
return {
"_id": that._id,
"indexing": that._indexing,
"free": that._free,
"location": that._location,
"database": that._database
};
};
that.check();
that._indexing_object = {};
that._indexing.forEach(function (meta_key) {
that._indexing_object[meta_key] = true;
});
}
/**
* Converts the indices file into an object usable by complex queries
* @method constructQueryObject
* @param {object} indices The index file
* @returns {object} response The query object constructed from Index file
*/
priv.constructQueryObject = function (indices, query_syntax) {
var j, k, l, m, n, use_index, index,
index_name, field_names, field, key, element,
query_index, query_object = [], field_name,
entry;
// returns index-to-use|can-do-query|can-do-query-and-results
use_index = priv.findBestIndexForQuery(query_syntax);
if (use_index.length > 0) {
for (j = 0; j < use_index.length; j += 1) {
index = use_index[j];
// NOTED: the index could be used to:
// (a) get all document ids matching query
// (b) get all document ids and results (= run complex query on index)
// right now, only (b) is supported, because the complex query is
// a single step process. If it was possible to first get the
// relevant document ids, then get the results, the index could be
// used to do the first step plus use GET on the returned documents
if (index.search && index.results) {
index_name = use_index[j].name;
query_index = indices[index_name];
// get fieldnames from this index
for (k = 0; k < priv.indices.length; k += 1) {
if (priv.indices[k].name === use_index[j].name) {
field_names = priv.indices[k].fields;
}
}
for (l = 0; l < field_names.length; l += 1) {
field_name = field_names[l];
// loop entries for this field name
field = query_index[field_name];
for (key in field) {
if (field.hasOwnProperty(key)) {
element = field[key];
// key can be "string" or "number" right now
if (priv.field_types[field_name] === "number") {
key = +key;
}
for (m = 0; m < element.length; m += 1) {
if (priv.searchIndexByValue(
query_object,
element[m],
"bool"
)) {
// loop object
for (n = 0; n < query_object.length; n += 1) {
entry = query_object[n];
if (entry.id === element[m]) {
entry[field_name] = key;
}
}
} else {
entry = {};
entry.id = element[m];
entry[field_name] = key;
query_object.push(entry);
}
}
}
}
}
}
}
}
return query_object;
};
/**
* Build the alldocs response from the index file (overriding substorage)
* @method allDocsResponseFromIndex
* @param {object} command The JIO command
* @param {boolean} include_docs Whether to also supply the document
* @param {object} option The options set for this method
* @returns {object} response The allDocs response
* The JIO index storage constructor
*/
priv.allDocsResponseFromIndex = function (indices, include_docs, option) {
var i, j, k, m, n = 0, l = priv.indices.length,
index, key, obj, prop, found, file, label,
unique_count = 0, unique_docids = [], all_doc_response = {},
success = function (content) {
file = { value: {} };
file.id = unique_docids[n];
file.key = unique_docids[n];
file.doc = content;
all_doc_response.rows.push(file);
// async counter, must be in callback
n += 1;
if (n === unique_count) {
that.success(all_doc_response);
}
},
error = function () {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Cannot find the document",
"reason": "Cannot get a document from substorage"
});
return;
function indexStorage(spec, my) {
var that, priv = {};
that = my.basicStorage(spec, my);
priv.indices = spec.indices;
priv.sub_storage = spec.sub_storage;
// Overrides
that.specToStore = function () {
return {
"indices": priv.indices,
"sub_storage": priv.sub_storage
};
};
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
// a lot of loops, not sure this is the fastest way
// loop index fields
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
index.current_field = index.current[label];
index.current_size = priv.getObjectSize(index.current_field);
// loop field id array
for (j = 0; j < index.current_size; j += 1) {
for (key in index.current_field) {
if (index.current_field.hasOwnProperty(key)) {
obj = index.current_field[key];
for (prop in obj) {
if (obj.hasOwnProperty(prop)) {
for (k = 0; k < unique_docids.length; k += 1) {
if (obj[prop] === unique_docids[k]) {
found = true;
break;
}
}
if (!found) {
unique_docids.push(obj[prop]);
unique_count += 1;
}
}
}
}
/**
* Return the similarity percentage (1 >= p >= 0) between two index lists.
*
* @method similarityPercentage
* @param {Array} list_a An index list
* @param {Array} list_b Another index list
* @return {Number} The similarity percentage
*/
priv.similarityPercentage = function (list_a, list_b) {
var ai, bi, count = 0;
for (ai = 0; ai < list_a.length; ai += 1) {
for (bi = 0; bi < list_b.length; bi += 1) {
if (list_a[ai] === list_b[bi]) {
count += 1;
}
}
}
}
return count / (list_a.length > list_b.length ?
list_a.length : list_b.length);
};
// construct allDocs response
all_doc_response.total_rows = unique_count;
all_doc_response.rows = [];
for (m = 0; m < unique_count; m += 1) {
// include_docs
if (include_docs) {
that.addJob(
"get",
priv.substorage,
unique_docids[m],
option,
success,
error
);
} else {
file = { value: {} };
file.id = unique_docids[m];
file.key = unique_docids[m];
all_doc_response.rows.push(file);
if (m === (unique_count - 1)) {
return all_doc_response;
/**
* Select the good index to use according to a select list.
*
* @method selectIndex
* @param {Array} select_list An array of strings
* @return {Number} The index index
*/
priv.selectIndex = function (select_list) {
var i, tmp, selector = {"index": 0, "similarity": 0};
for (i = 0; i < priv.indices.length; i += 1) {
tmp = priv.similarityPercentage(select_list,
priv.indices[i].index);
if (tmp > selector.similarity) {
selector.index = i;
selector.similarity = tmp;
}
}
}
};
return selector.index;
};
/**
* Post document to substorage and create/update index file(s)
* @method post
* @param {object} command The JIO command
* @param {string} source The source of the function call
*/
priv.postOrPut = function (command, source) {
var f = {}, indices, doc;
doc = command.cloneDoc();
if (typeof doc._id !== "string") {
doc._id = priv.generateUuid();
}
f.getIndices = function () {
var option = command.cloneOption();
/**
* Get a database
*
* @method getIndexDatabase
* @param {Object} option The command option
* @param {Number} number The location in priv.indices
* @param {Function} callback The callback
*/
priv.getIndexDatabase = function (option, number, callback) {
that.addJob(
"get",
priv.substorage,
{"_id": priv.index_suffix},
priv.indices[number].sub_storage || priv.sub_storage,
{"_id": priv.indices[number].id},
option,
function (response) {
indices = response;
f.postDocument("put");
callback(new JSONIndex(response));
},
function (err) {
switch (err.status) {
case 404:
if (source !== 'PUTATTACHMENT') {
indices = priv.createEmptyIndexArray();
f.postDocument("post");
} else {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not found",
"message": "Document not found",
"reason": "Document not found"
});
return;
}
break;
default:
err.message = "Cannot retrieve index array";
that.error(err);
break;
if (err.status === 404) {
callback(new JSONIndex({
"_id": priv.indices[number].id,
"indexing": priv.indices[number].index
}));
return;
}
err.message = "Unable to get index database.";
that.error(err);
}
);
};
f.postDocument = function (index_update_method) {
if (priv.isDocidInIndex(indices, doc) && source === 'POST') {
// POST the document already exists
that.error({
"status": 409,
"statusText": "Conflicts",
"error": "conflicts",
"message": "Cannot create a new document",
"reason": "Document already exists"
});
return;
/**
* Gets a list containing all the databases set in the storage description.
*
* @method getIndexDatabaseList
* @param {Object} option The command option
* @param {Function} callback The result callback(database_list)
*/
priv.getIndexDatabaseList = function (option, callback) {
var i, count = 0, callbacks = {}, response_list = [];
callbacks.error = function (index) {
return function (err) {
if (err.status === 404) {
response_list[index] = new JSONIndex({
"_id": priv.indices[index].id,
"indexing": priv.indices[index].index
});
count += 1;
if (count === priv.indices.length) {
callback(response_list);
}
return;
}
err.message = "Unable to get index database.";
that.error(err);
};
};
callbacks.success = function (index) {
return function (response) {
response_list[index] = new JSONIndex(response);
count += 1;
if (count === priv.indices.length) {
callback(response_list);
}
};
};
for (i = 0; i < priv.indices.length; i += 1) {
that.addJob(
"get",
priv.indices[i].sub_storage || priv.sub_storage,
{"_id": priv.indices[i].id},
option,
callbacks.success(i),
callbacks.error(i)
);
}
if (source !== 'PUTATTACHMENT') {
indices = priv.updateIndices(indices, doc);
};
/**
* Saves all the databases to the remote(s).
*
* @method storeIndexDatabaseList
* @param {Array} database_list The database list
* @param {Object} option The command option
* @param {Function} callback The result callback(err, response)
*/
priv.storeIndexDatabaseList = function (database_list, option, callback) {
var i, count = 0, count_max = 0, onResponse, onError;
onResponse = function (response) {
count += 1;
if (count === count_max) {
callback({"ok": true});
}
};
onError = function (err) {
err.message = "Unable to store index database.";
that.error(err);
};
for (i = 0; i < priv.indices.length; i += 1) {
if (database_list[i] !== undefined) {
count_max += 1;
that.addJob(
"put",
priv.indices[i].sub_storage || priv.sub_storage,
database_list[i].serialized(),
option,
onResponse,
onError
);
}
}
};
/**
* A generic request method which delegates the request to the sub storage.
* On response, it will index the document from the request and update all
* the databases.
*
* @method genericRequest
* @param {Command} command The JIO command
* @param {Function} method The request method
*/
priv.genericRequest = function (command, method) {
var doc = command.cloneDoc(), option = command.cloneOption();
that.addJob(
source === 'PUTATTACHMENT' ? "putAttachment" : "post",
priv.substorage,
method,
priv.sub_storage,
doc,
command.cloneOption(),
function () {
if (source !== 'PUTATTACHMENT') {
f.sendIndices(index_update_method);
} else {
that.success({
"ok": true,
"id": doc._id,
"attachment": doc._attachment
});
}
},
function (err) {
switch (err.status) {
case 409:
// file already exists
if (source !== 'PUTATTACHMENT') {
f.sendIndices(index_update_method);
} else {
that.success({
"ok": true,
"id": doc._id
option,
function (response) {
switch (method) {
case "post":
case "put":
case "remove":
doc._id = response.id;
priv.getIndexDatabaseList(option, function (database_list) {
var i;
switch (method) {
case "post":
case "put":
for (i = 0; i < database_list.length; i += 1) {
database_list[i].put(doc);
}
break;
case "remove":
for (i = 0; i < database_list.length; i += 1) {
database_list[i].remove(doc);
}
break;
default:
break;
}
priv.storeIndexDatabaseList(database_list, option, function () {
that.success({"ok": true, "id": doc._id});
});
}
});
break;
default:
err.message = "Cannot upload document";
that.error(err);
that.success(response);
break;
}
}
);
};
f.sendIndices = function (method) {
indices._id = priv.index_suffix;
that.addJob(
method,
priv.substorage,
indices,
command.cloneOption(),
function () {
that.success({
"ok": true,
"id": doc._id
});
},
function (err) {
// xxx do we try to delete the posted document ?
err.message = "Cannot save index file";
that.error(err);
return that.error(err);
}
);
};
f.getIndices();
};
/**
* Update the document metadata and update the index
* @method put
* @param {object} command The JIO command
*/
that.post = function (command) {
priv.postOrPut(command, 'POST');
};
/**
* Post the document metadata and update the index
* @method post
* @param {object} command The JIO command
*/
that.post = function (command) {
priv.genericRequest(command, 'post');
};
/**
* Update the document metadata and update the index
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.postOrPut(command, 'PUT');
};
/**
* Update the document metadata and update the index
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.genericRequest(command, 'put');
};
/**
* Add an attachment to a document (no index modification)
* @method putAttachment
* @param {object} command The JIO command
*/
that.putAttachment = function (command) {
priv.postOrPut(command, 'PUTATTACHMENT');
};
/**
* Add an attachment to a document (no index modification)
* @method putAttachment
* @param {object} command The JIO command
*/
that.putAttachment = function (command) {
priv.genericRequest(command, 'putAttachment');
};
/**
* Get the document metadata
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
that.addJob(
"get",
priv.substorage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
that.success(response);
},
function (err) {
that.error(err);
}
);
};
/**
* Get the document metadata
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
priv.genericRequest(command, 'get');
};
/**
* Get the attachment.
* @method getAttachment
* @param {object} command The JIO command
*/
that.getAttachment = function (command) {
that.addJob(
"getAttachment",
priv.substorage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
that.success(response);
},
function (err) {
that.error(err);
}
);
};
/**
* Get the attachment.
* @method getAttachment
* @param {object} command The JIO command
*/
that.getAttachment = function (command) {
priv.genericRequest(command, 'getAttachment');
};
/**
* Remove document - removing documents updates index!.
* @method remove
* @param {object} command The JIO command
*/
that.remove = function (command) {
var f = {}, indices, doc, docid, option;
/**
* Remove document - removing documents updates index!.
* @method remove
* @param {object} command The JIO command
*/
that.remove = function (command) {
priv.genericRequest(command, 'remove');
};
doc = command.cloneDoc();
option = command.cloneOption();
/**
* Remove attachment
* @method removeAttachment
* @param {object} command The JIO command
*/
that.removeAttachment = function (command) {
priv.genericRequest(command, 'removeAttachment');
};
f.removeDocument = function (type) {
that.addJob(
"remove",
priv.substorage,
doc,
option,
function (response) {
that.success(response);
},
function () {
that.error({
"status": 409,
"statusText": "Conflict",
"error": "conflict",
"message": "Document Update Conflict",
"reason": "Could not delete document or attachment"
});
/**
* Gets a document list from the substorage
* Options:
* - {boolean} include_docs Also retrieve the actual document content.
* @method allDocs
* @param {object} command The JIO command
*/
that.allDocs = function (command) {
var option = command.cloneOption(),
index = priv.selectIndex(option.select_list || []);
// Include docs option is ignored, if you want to get all the document,
// don't use index storage!
option.select_list = option.select_list || [];
option.select_list.push("_id");
priv.getIndexDatabase(option, index, function (db) {
var i, id;
db = db._database;
complex_queries.QueryFactory.create(option.query || '').
exec(db, option);
for (i = 0; i < db.length; i += 1) {
id = db[i]._id;
delete db[i]._id;
db[i] = {
"id": id,
"key": id,
"value": db[i],
};
}
);
that.success({"total_rows": db.length, "rows": db});
});
};
that.check = function (command) {
that.repair(command, true);
};
f.getIndices = function () {
priv.repairIndexDatabase = function (command, index, just_check) {
var i, option = command.cloneOption();
that.addJob(
"get",
priv.substorage,
{"_id": priv.index_suffix},
option,
'allDocs',
priv.sub_storage,
{},
{'include_docs': true},
function (response) {
// if deleting an attachment
if (typeof command.getAttachmentId() === 'string') {
f.removeDocument('attachment');
} else {
indices = priv.cleanIndices(response, doc);
// store update index file
that.addJob(
"put",
priv.substorage,
indices,
command.cloneOption(),
function () {
// remove actual document
f.removeDocument('doc');
},
function (err) {
err.message = "Cannot save index file";
that.error(err);
var db_list = [], db = new JSONIndex({
"_id": command.getDocId(),
"indexing": priv.indices[index].index
});
for (i = 0; i < response.rows.length; i += 1) {
db.put(response.rows[i].doc);
}
db_list[index] = db;
if (just_check) {
priv.getIndexDatabase(option, index, function (current_db) {
if (db.equals(current_db)) {
return that.success({"ok": true, "_id": command.getDocId()});
}
);
return that.error(generateErrorObject(
"Different Index",
"Check failed",
"corrupt index database"
));
});
} else {
priv.storeIndexDatabaseList(db_list, {}, function () {
that.success({"ok": true, "_id": command.getDocId()});
});
}
},
function () {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Document index not found, please check document ID",
"reason": "Incorrect document ID"
});
return;
function (err) {
err.message = "Unable to repair the index database";
that.error(err);
}
);
};
f.getIndices();
};
/**
* Remove document - removing documents updates index!.
* @method remove
* @param {object} command The JIO command
*/
that.removeAttachment = function (command) {
var f = {}, indices, doc, docid, option;
doc = command.cloneDoc();
option = command.cloneOption();
f.removeDocument = function (type) {
that.addJob(
"removeAttachment",
priv.substorage,
doc,
option,
that.success,
that.error
);
};
f.getIndices = function () {
priv.repairDocument = function (command, just_check) {
var i, option = command.cloneOption();
that.addJob(
"get",
priv.substorage,
{"_id": priv.index_suffix},
option,
priv.sub_storage,
command.cloneDoc(),
{},
function (response) {
// if deleting an attachment
if (typeof command.getAttachmentId() === 'string') {
f.removeDocument('attachment');
} else {
indices = priv.cleanIndices(response, doc);
// store update index file
that.addJob(
"put",
priv.substorage,
indices,
command.cloneOption(),
function () {
// remove actual document
f.removeDocument('doc');
},
function (err) {
err.message = "Cannot save index file";
that.error(err);
response._id = command.getDocId();
priv.getIndexDatabaseList(option, function (database_list) {
if (just_check) {
for (i = 0; i < database_list.length; i += 1) {
try {
database_list[i].checkDocument(response);
} catch (e) {
return that.error(generateErrorObject(
e.message,
"Check failed",
"corrupt index database"
));
}
}
);
}
that.success({"_id": command.getDocId(), "ok": true});
} else {
for (i = 0; i < database_list.length; i += 1) {
database_list[i].put(response);
}
priv.storeIndexDatabaseList(database_list, option, function () {
that.success({"ok": true, "id": command.getDocId()});
});
}
});
},
function (err) {
that.error(err);
err.message = "Unable to repair document";
return that.error(err);
}
);
};
f.getIndices();
};
/**
* Gets a document list from the substorage
* Options:
* - {boolean} include_docs Also retrieve the actual document content.
* @method allDocs
* @param {object} command The JIO command
*/
//{
// "total_rows": 4,
// "rows": [
// {
// "id": "otherdoc",
// "key": "otherdoc",
// "value": {
// "rev": "1-3753476B70A49EA4D8C9039E7B04254C"
// }
// },{...}
// ]
//}
that.allDocs = function (command) {
var f = {}, option, all_docs_response, query_object, query_syntax,
query_response;
option = command.cloneOption();
f.getIndices = function () {
that.repair = function (command, just_check) {
var database_index = -1, i;
for (i = 0; i < priv.indices.length; i += 1) {
if (priv.indices[i].id === command.getDocId()) {
database_index = i;
break;
}
}
that.addJob(
"get",
priv.substorage,
{"_id": priv.index_suffix},
option,
"repair",
priv.sub_storage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
query_syntax = command.getOption('query');
if (query_syntax !== undefined) {
// build complex query object
query_object = priv.constructQueryObject(response, query_syntax);
if (query_object.length === 0) {
that.addJob(
"allDocs",
priv.substorage,
undefined,
option,
that.success,
that.error
);
} else {
// we can use index, run query on index
query_response =
jIO.ComplexQueries.query(query_syntax, query_object);
that.success(query_response);
}
} else if (command.getOption('include_docs')) {
priv.allDocsResponseFromIndex(response, true, option);
if (database_index !== -1) {
priv.repairIndexDatabase(command, database_index, just_check);
} else {
all_docs_response =
priv.allDocsResponseFromIndex(response, false, option);
that.success(all_docs_response);
priv.repairDocument(command, just_check);
}
},
that.error
function (err) {
err.message = "Could not repair sub storage";
that.error(err);
}
);
};
f.getIndices();
};
return that;
});
return that;
}
if (typeof exports === "object") {
// nodejs export module
Object.defineProperty(exports, "jio_index_storage", {
configurable: false,
enumerable: true,
writable: false,
value: indexStorage
});
} else if (typeof define === "function" && define.amd) {
// requirejs export
define(indexStorage);
} else {
// classical browser and web workers JIO export
jIO.addStorageType("indexed", indexStorage);
}
}());
src/jio.storage/localstorage.js
View file @ 4a6e56c2
......@@ -3,8 +3,11 @@
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global jIO: true, localStorage: true, setTimeout: true */
/*global jIO: true, localStorage: true, setTimeout: true,
complex_queries: true */
/**
* JIO Local Storage. Type = 'local'.
* Local browser "database" storage.
......@@ -134,9 +137,11 @@ jIO.addStorageType('local', function (spec, my) {
}
doc = localstorage.getItem(priv.localpath + "/" + doc_id);
if (doc === null) {
doc = command.cloneDoc();
doc._id = doc_id;
// the document does not exist
localstorage.setItem(priv.localpath + "/" + doc_id,
command.cloneDoc());
doc);
that.success({
"ok": true,
"id": doc_id
......@@ -359,47 +364,66 @@ jIO.addStorageType('local', function (spec, my) {
* @param {object} command The JIO command
*/
that.allDocs = function (command) {
var i, j, file, items = 0,
s = new RegExp("^" + priv.localpath + "\\/[^/]+$"),
all_doc_response = {},
query_object = [], query_syntax, query_response = [];
query_syntax = command.getOption('query');
if (query_syntax === undefined) {
all_doc_response.rows = [];
var i, row, path_re, rows = [], document_list = [], option, document_object;
path_re = new RegExp(
"^" + complex_queries.stringEscapeRegexpCharacters(priv.localpath) +
"/[^/]+$"
);
option = command.cloneOption();
if (typeof complex_queries !== "object" ||
(option.query === undefined && option.sort_on === undefined &&
option.select_list === undefined &&
option.include_docs === undefined)) {
rows = [];
for (i in localStorage) {
if (localStorage.hasOwnProperty(i)) {
// filter non-documents
if (s.test(i)) {
items += 1;
j = i.split('/').slice(-1)[0];
file = { value: {} };
file.id = j;
file.key = j;
if (path_re.test(i)) {
row = { value: {} };
row.id = i.split('/').slice(-1)[0];
row.key = row.id;
if (command.getOption('include_docs')) {
file.doc = JSON.parse(localStorage.getItem(i));
row.doc = JSON.parse(localStorage.getItem(i));
}
all_doc_response.rows.push(file);
rows.push(row);
}
}
}
all_doc_response.total_rows = items;
that.success(all_doc_response);
that.success({"rows": rows, "total_rows": rows.length});
} else {
// create complex query object from returned results
for (i in localStorage) {
if (localStorage.hasOwnProperty(i)) {
if (s.test(i)) {
items += 1;
j = i.split('/').slice(-1)[0];
query_object.push(localstorage.getItem(i));
if (path_re.test(i)) {
document_list.push(localstorage.getItem(i));
}
}
}
query_response = jIO.ComplexQueries.query(query_syntax, query_object);
that.success(query_response);
option.select_list = option.select_list || [];
option.select_list.push("_id");
if (option.include_docs === true) {
document_object = {};
document_list.forEach(function (meta) {
document_object[meta._id] = meta;
});
}
complex_queries.QueryFactory.create(option.query || "").
exec(document_list, option);
document_list = document_list.map(function (value) {
var o = {
"id": value._id,
"key": value._id
};
if (option.include_docs === true) {
o.doc = document_object[value._id];
delete document_object[value._id];
}
delete value._id;
o.value = value;
return o;
});
that.success({"total_rows": document_list.length,
"rows": document_list});
}
};
......
src/queries/begin.js
View file @ 4a6e56c2
......@@ -4,11 +4,27 @@
* http://www.gnu.org/licenses/lgpl.html
*/
(function (scope) {
/**
* Provides some function to use complex queries with item list
*
* @module complex_queries
*/
var complex_queries;
(function () {
"use strict";
Object.defineProperty(scope, "ComplexQueries", {
configurable: false,
enumerable: false,
writable: false,
value: {}
});
var to_export = {}, module_name = "complex_queries";
/**
* Add a secured (write permission denied) property to an object.
*
* @param {Object} object The object to fill
* @param {String} key The object key where to store the property
* @param {Any} value The value to store
*/
function _export(key, value) {
Object.defineProperty(to_export, key, {
"configurable": false,
"enumerable": true,
"writable": false,
"value": value
});
}
src/queries/complexquery.js 0 → 100644
View file @ 4a6e56c2
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global newClass: true, Query: true, query_class_dict: true,
_export: true, QueryFactory: true */
/**
* The ComplexQuery inherits from Query, and compares one or several metadata
* values.
*
* @class ComplexQuery
* @extends Query
* @param {Object} [spec={}] The specifications
* @param {String} [spec.operator="AND"] The compare method to use
* @param {String} spec.key The metadata key
* @param {String} spec.value The value of the metadata to compare
*/
var ComplexQuery = newClass(Query, function (spec) {
/**
* Logical operator to use to compare object values
*
* @attribute operator
* @type String
* @default "AND"
* @optional
*/
this.operator = spec.operator || "AND";
/**
* The sub Query list which are used to query an item.
*
* @attribute query_list
* @type Array
* @default []
* @optional
*/
this.query_list = spec.query_list || [];
this.query_list = this.query_list.map(QueryFactory.create);
/**
* #crossLink "Query/match:method"
*/
this.match = function (item, wildcard_character) {
return this[this.operator](item, wildcard_character);
};
/**
* #crossLink "Query/toString:method"
*/
this.toString = function () {
var str_list = ["("], this_operator = this.operator;
this.query_list.forEach(function (query) {
str_list.push(query.toString());
str_list.push(this_operator);
});
str_list.pop(); // remove last operator
str_list.push(")");
return str_list.join(" ");
};
/**
* #crossLink "Query/serialized:method"
*/
this.serialized = function () {
var s = {
"type": "complex",
"operator": this.operator,
"query_list": []
};
this.query_list.forEach(function (query) {
s.query_list.push(query.serialized());
});
return s;
};
/**
* Comparison operator, test if all sub queries match the
* item value
*
* @method AND
* @param {Object} item The item to match
* @param {String} wildcard_character The wildcard character
* @return {Boolean} true if all match, false otherwise
*/
this.AND = function (item, wildcard_character) {
var i;
for (i = 0; i < this.query_list.length; i += 1) {
if (!this.query_list[i].match(item, wildcard_character)) {
return false;
}
}
return true;
};
/**
* Comparison operator, test if one of the sub queries matches the
* item value
*
* @method OR
* @param {Object} item The item to match
* @param {String} wildcard_character The wildcard character
* @return {Boolean} true if one match, false otherwise
*/
this.OR = function (item, wildcard_character) {
var i;
for (i = 0; i < this.query_list.length; i += 1) {
if (this.query_list[i].match(item, wildcard_character)) {
return true;
}
}
return false;
};
/**
* Comparison operator, test if the sub query does not match the
* item value
*
* @method NOT
* @param {Object} item The item to match
* @param {String} wildcard_character The wildcard character
* @return {Boolean} true if one match, false otherwise
*/
this.NOT = function (item, wildcard_character) {
return !this.query_list[0].match(item, wildcard_character);
};
});
query_class_dict.complex = ComplexQuery;
_export("ComplexQuery", ComplexQuery);
src/queries/end.js
View file @ 4a6e56c2
}(jIO));
if (typeof define === "function" && define.amd) {
define(to_export);
} else if (typeof window === "object") {
Object.defineProperty(window, module_name, {
configurable: false,
enumerable: true,
writable: false,
value: to_export
});
} else if (typeof exports === "object") {
var i;
for (i in to_export) {
if (to_export.hasOwnProperty(i)) {
exports[i] = to_export[i];
}
}
} else {
complex_queries = to_export;
}
}());
src/queries/parser-begin.js
View file @ 4a6e56c2
Object.defineProperty(scope.ComplexQueries, "parse", {
configurable: false,
enumerable: false,
writable: false,
value: function (string) {
function parseStringToObject(string) {
src/queries/parser-end.js
View file @ 4a6e56c2
return result;
}
});
} // parseStringToObject
src/queries/parser.par
View file @ 4a6e56c2
......@@ -37,7 +37,7 @@ boolean_expression
expression
: LEFT_PARENTHESE search_text RIGHT_PARENTHESE [* %% = %2; *]
| COLUMN expression [* simpleQuerySetId(%2,%1.split(':').slice(0,-1).join(':')); %% = %2; *]
| COLUMN expression [* simpleQuerySetKey(%2,%1.split(':').slice(0,-1).join(':')); %% = %2; *]
| value [* %% = %1; *]
;
......@@ -53,59 +53,60 @@ string
[*
var arrayExtend = function () {
var j,i,newlist=[],listoflists = arguments;
for (j=0; j<listoflists.length; ++j) {
for (i=0; i<listoflists[j].length; ++i) {
newlist.push(listoflists[j][i]);
}
var j, i, newlist = [], list_list = arguments;
for (j = 0; j < list_list.length; j += 1) {
for (i = 0; i < list_list[j].length; i += 1) {
newlist.push(list_list[j][i]);
}
return newlist;
};
var mkSimpleQuery = function (id,value,operator) {
return {type:'simple',operator:'=',id:id,value:value};
};
var mkNotQuery = function (query) {
if (query.operator === 'NOT') {
return query.query_list[0];
}
return {type:'complex',operator:'NOT',query_list:[query]};
};
var mkComplexQuery = function (operator,query_list) {
var i,query_list2 = [];
for (i=0; i<query_list.length; ++i) {
if (query_list[i].operator === operator) {
query_list2 = arrayExtend(query_list2,query_list[i].query_list);
} else {
query_list2.push(query_list[i]);
}
}
return {type:'complex',operator:operator,query_list:query_list2};
};
var simpleQuerySetId = function (query, id) {
var i;
if (query.type === 'complex') {
for (i = 0; i < query.query_list.length; ++i) {
simpleQuerySetId (query.query_list[i],id);
}
return true;
}
if (query.type === 'simple' && !query.id) {
query.id = id;
return true;
}
return newlist;
}, mkSimpleQuery = function (key, value, operator) {
return {"type": "simple", "operator": "=", "key": key, "value": value};
}, mkNotQuery = function (query) {
if (query.operator === "NOT") {
return query.query_list[0];
}
return {"type": "complex", "operator": "NOT", "query_list": [query]};
}, mkComplexQuery = function (operator, query_list) {
var i, query_list2 = [];
for (i = 0; i < query_list.length; i += 1) {
if (query_list[i].operator === operator) {
query_list2 = arrayExtend(query_list2, query_list[i].query_list);
} else {
query_list2.push(query_list[i]);
}
return false;
};
var error_offsets = [];
var error_lookaheads = [];
var error_count = 0;
var result;
if ( ( error_count = __##PREFIX##parse( string, error_offsets, error_lookaheads ) ) > 0 ) {
var i;
for (i = 0; i < error_count; ++i) {
throw new Error ( "Parse error near \"" +
string.substr ( error_offsets[i] ) +
"\", expecting \"" +
error_lookaheads[i].join() + "\"" );
}
return {type:"complex",operator:operator,query_list:query_list2};
}, simpleQuerySetKey = function (query, key) {
var i;
if (query.type === "complex") {
for (i = 0; i < query.query_list.length; ++i) {
simpleQuerySetKey (query.query_list[i],key);
}
return true;
}
if (query.type === "simple" && !query.key) {
query.key = key;
return true;
}
return false;
},
error_offsets = [],
error_lookaheads = [],
error_count = 0,
result;
if ((error_count = __##PREFIX##parse(string, error_offsets, error_lookaheads)) > 0) {
var i;
for (i = 0; i < error_count; i += 1) {
throw new Error("Parse error near \"" +
string.substr(error_offsets[i]) +
"\", expecting \"" +
error_lookaheads[i].join() + "\"");
}
}
*]
src/queries/query.js
View file @ 4a6e56c2
Object.defineProperty(scope.ComplexQueries,"query",{
configurable:false,enumerable:false,writable:false,
value: function (query, object_list) {
var wildcard_character = typeof query.wildcard_character === 'string' ?
query.wildcard_character : '%',
operator_actions = {
'=': function (value1, value2) {
value1 = '' + value1;
return value1.match (convertToRegexp (
value2, wildcard_character
)) || false && true;
},
'!=': function (value1, value2) {
value1 = '' + value1;
return !(value1.match (convertToRegexp (
value2, wildcard_character
)));
},
'<': function (value1, value2) { return value1 < value2; },
'<=': function (value1, value2) { return value1 <= value2; },
'>': function (value1, value2) { return value1 > value2; },
'>=': function (value1, value2) { return value1 >= value2; },
'AND': function (item, query_list) {
var i;
for (i=0; i<query_list.length; ++i) {
if (! itemMatchesQuery (item, query_list[i])) {
return false;
}
}
return true;
},
'OR': function (item, query_list) {
var i;
for (i=0; i<query_list.length; ++i) {
if (itemMatchesQuery (item, query_list[i])) {
return true;
}
}
return false;
},
'NOT': function (item, query_list) {
return !itemMatchesQuery(item, query_list[0]);
}
},
convertToRegexp = function (string) {
return subString('^' + string.replace(
new RegExp(
'([\\{\\}\\(\\)\\^\\$\\&\\.\\*\\?\\\/\\+\\|\\[\\]\\-\\\\])'.
replace (wildcard_character?
'\\'+wildcard_character:undefined,''),
'g'
),
'\\$1'
) + '$',(wildcard_character||undefined), '.*');
},
subString = function (string, substring, newsubstring) {
var res = '', i = 0;
if (substring === undefined) {
return string;
}
while (1) {
var tmp = string.indexOf(substring,i);
if (tmp === -1) {
break;
}
for (; i < tmp; ++i) {
res += string[i];
}
res += newsubstring;
i += substring.length;
}
for (; i<string.length; ++i) {
res += string[i];
}
return res;
},
itemMatchesQuery = function (item, query_object) {
var i;
if (query_object.type === 'complex') {
return operator_actions[query_object.operator](
item, query_object.query_list
);
} else {
if (query_object.id) {
if (typeof item[query_object.id] !== 'undefined') {
return operator_actions[query_object.operator](
item[query_object.id], query_object.value
);
} else {
return false;
}
} else {
return true;
}
}
},
select = function (list, select_list) {
var i;
if (select_list.length === 0) {
return;
}
for (i=0; i<list.length; ++i) {
var list_value = {}, k;
for (k=0; k<select_list.length; ++k) {
list_value[select_list[k]] =
list[i][select_list[k]];
}
list[i] = list_value;
}
},
sortFunction = function (key, asc) {
if (asc === 'descending') {
return function (a,b) {
return a[key] < b[key] ? 1 : a[key] > b[key] ? -1 : 0;
};
}
return function (a,b) {
return a[key] > b[key] ? 1 : a[key] < b[key] ? -1 : 0;
};
},
mergeList = function (list, list_to_merge, index) {
var i,j;
for (i = index,j = 0; i < list_to_merge.length + index; ++i, ++j) {
list[i] = list_to_merge[j];
}
},
sort = function (list, sort_list) {
var i, tmp, key, asc, sortAndMerge = function() {
sort(tmp,sort_list.slice(1));
mergeList(list,tmp,i-tmp.length);
tmp = [list[i]];
};
if (list.length < 2) {
return;
}
if (sort_list.length === 0) {
return;
}
key = sort_list[0][0];
asc = sort_list[0][1];
list.sort (sortFunction (key,asc));
tmp = [list[0]];
for (i = 1; i < list.length; ++i) {
if (tmp[0][key] === list[i][key]) {
tmp.push(list[i]);
} else {
sortAndMerge();
}
}
sortAndMerge();
},
limit = function (list, limit_list) {
var i;
if (typeof limit_list[0] !== 'undefined') {
if (typeof limit_list[1] !== 'undefined') {
if (list.length > limit_list[1] + limit_list[0]) {
list.length = limit_list[1] + limit_list[0];
}
list.splice(0,limit_list[0]);
} else {
list.length = limit_list[0];
}
}
},
////////////////////////////////////////////////////////////
result_list = [], result_list_tmp = [], j;
object_list = object_list || [];
if (query.query === undefined) {
result_list = object_list;
} else {
for (j=0; j<object_list.length; ++j) {
if ( itemMatchesQuery (
object_list[j], scope.ComplexQueries.parse (query.query)
)) {
result_list.push(object_list[j]);
}
}
}
if (query.filter) {
select(result_list,query.filter.select_list || []);
sort(result_list,query.filter.sort_on || []);
limit(result_list,query.filter.limit || []);
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global newClass: true, sortFunction: true, parseStringToObject: true,
_export: true, stringEscapeRegexpCharacters: true */
/**
* The query to use to filter a list of objects.
* This is an abstract class.
*
* @class Query
* @constructor
*/
var Query = newClass(function () {
var that = this, emptyFunction = function () {};
/**
* Filter the item list with matching item only
*
* @method exec
* @param {Array} item_list The list of object
* @param {Object} [option] Some operation option
* @param {String} [option.wildcard_character="%"] The wildcard character
* @param {Array} [option.select_list] A object keys to retrieve
* @param {Array} [option.sort_on] Couples of object keys and "ascending"
* or "descending"
* @param {Array} [option.limit] Couple of integer, first is an index and
* second is the length.
*/
that.exec = function (item_list, option) {
var i = 0;
while (i < item_list.length) {
if (!that.match(item_list[i], option.wildcard_character)) {
item_list.splice(i, 1);
} else {
i += 1;
}
}
if (option.sort_on) {
Query.sortOn(option.sort_on, item_list);
}
if (option.limit) {
item_list.splice(0, option.limit[0]);
if (option.limit[1]) {
item_list.splice(option.limit[1]);
}
}
Query.filterListSelect(option.select_list || [], item_list);
};
/**
* Test if an item matches this query
*
* @method match
* @param {Object} item The object to test
* @return {Boolean} true if match, false otherwise
*/
that.match = function (item, wildcard_character) {
return true;
};
/**
* The recursive parser.
*
* @method recParse
* @private
* @param {Object} object The object shared in the parse process
* @param {Object} options Some options usable in the parseMethods
* @return {Any} The parser result
*/
function recParse(object, option) {
var i, query = object.parsed;
if (query.type === "complex") {
for (i = 0; i < query.query_list.length; i += 1) {
object.parsed = query.query_list[i];
recParse(object, option);
query.query_list[i] = object.parsed;
}
object.parsed = query;
that.onParseComplexQuery(object, option);
} else if (query.type === "simple") {
that.onParseSimpleQuery(object, option);
}
}
/**
* Browse the Query in deep calling parser method in each step.
*
* `onParseStart` is called first, on end `onParseEnd` is called.
* It starts from the simple queries at the bottom of the tree calling the
* parser method `onParseSimpleQuery`, and go up calling the
* `onParseComplexQuery` method.
*
* @method parse
* @param {Object} option Any options you want (except 'parsed')
* @return {Any} The parse result
*/
that.parse = function (option) {
var object;
object = {"parsed": JSON.parse(JSON.stringify(that.serialized()))};
that.onParseStart(object, option);
recParse(object, option);
that.onParseEnd(object, option);
return object.parsed;
};
/**
* Called before parsing the query. Must be overridden!
*
* @method onParseStart
* @param {Object} object The object shared in the parse process
* @param {Object} option Some option gave in parse()
*/
that.onParseStart = emptyFunction;
/**
* Called when parsing a simple query. Must be overridden!
*
* @method onParseSimpleQuery
* @param {Object} object The object shared in the parse process
* @param {Object} option Some option gave in parse()
*/
that.onParseSimpleQuery = emptyFunction;
/**
* Called when parsing a complex query. Must be overridden!
*
* @method onParseComplexQuery
* @param {Object} object The object shared in the parse process
* @param {Object} option Some option gave in parse()
*/
that.onParseComplexQuery = emptyFunction;
/**
* Called after parsing the query. Must be overridden!
*
* @method onParseEnd
* @param {Object} object The object shared in the parse process
* @param {Object} option Some option gave in parse()
*/
that.onParseEnd = emptyFunction;
/**
* Convert this query to a parsable string.
*
* @method toString
* @return {String} The string version of this query
*/
that.toString = function () {
return "";
};
/**
* Convert this query to an jsonable object in order to be remake thanks to
* QueryFactory class.
*
* @method serialized
* @return {Object} The jsonable object
*/
that.serialized = function () {
return undefined;
};
}, {"static_methods": {
/**
* Filter a list of items, modifying them to select only wanted keys.
*
* @method filterListSelect
* @static
* @param {Array} select_option Key list to keep
* @param {Array} list The item list to filter
*/
"filterListSelect": function (select_option, list) {
var i, j, new_item;
for (i = 0; i < list.length; i += 1) {
new_item = {};
for (j = 0; j < select_option.length; j += 1) {
new_item[select_option[j]] = list[i][select_option[j]];
}
for (j in new_item) {
if (new_item.hasOwnProperty(j)) {
list[i] = new_item;
break;
}
return result_list;
}
}
},
/**
* Sort a list of items, according to keys and directions.
*
* @method sortOn
* @static
* @param {Array} sort_on_option List of couples [key, direction]
* @param {Array} list The item list to sort
*/
"sortOn": function (sort_on_option, list) {
var sort_index;
for (sort_index = sort_on_option.length - 1; sort_index >= 0;
sort_index -= 1) {
list.sort(sortFunction(
sort_on_option[sort_index][0],
sort_on_option[sort_index][1]
));
}
});
},
/**
* Parse a text request to a json query object tree
*
* @method parseStringToObject
* @static
* @param {String} string The string to parse
* @return {Object} The json query tree
*/
"parseStringToObject": parseStringToObject,
/**
* Convert a search text to a regexp.
*
* @method convertStringToRegExp
* @static
* @param {String} string The string to convert
* @param {String} [wildcard_character=undefined] The wildcard chararter
* @return {RegExp} The search text regexp
*/
"convertStringToRegExp": function (string, wildcard_character) {
return new RegExp("^" + stringEscapeRegexpCharacters(string).replace(
stringEscapeRegexpCharacters(wildcard_character),
'.*'
) + "$");
}
}});
_export("Query", Query);
src/queries/queryfactory.js 0 → 100644
View file @ 4a6e56c2
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global _export: true, ComplexQuery: true, SimpleQuery: true,
newClass: true, Query: true */
var query_class_dict = {}, QueryFactory;
/**
* Provides static methods to create Query object
*
* @class QueryFactory
*/
QueryFactory = newClass({
"static_methods": {
/**
* Creates Query object from a search text string or a serialized version
* of a Query.
*
* @method create
* @static
* @param {Object,String} object The search text or the serialized version
* of a Query
* @return {Query} A Query object
*/
"create": function (object) {
if (object === "") {
return new Query();
}
if (typeof object === "string") {
object = Query.parseStringToObject(object);
}
if (typeof (object || {}).type === "string" &&
query_class_dict[object.type]) {
return new query_class_dict[object.type](object);
}
return null;
}
}
}, function () {});
_export("QueryFactory", QueryFactory);
src/queries/serializer.js
View file @ 4a6e56c2
Object.defineProperty(scope.ComplexQueries,"serialize",{
configurable:false,enumerable:false,writable:false,value:function(query){
var str_list = [], i;
if (query.type === 'complex') {
str_list.push ( '(' );
for (i=0; i<query.query_list.length; ++i) {
str_list.push( scope.ComplexQueries.serialize(query.query_list[i]) );
str_list.push( query.operator );
}
str_list.length --;
str_list.push ( ')' );
return str_list.join(' ');
} else if (query.type === 'simple') {
return query.id + (query.id?': ':'') + query.operator + ' "' + query.value + '"';
}
return query;
}
});
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global _export: true, to_export: true */
function objectToSearchText(query) {
var str_list = [];
if (query.type === "complex") {
str_list.push("(");
(query.query_list || []).forEach(function (sub_query) {
str_list.push(objectToSearchText(sub_query));
str_list.push(query.operator);
});
str_list.length -= 1;
str_list.push(")");
return str_list.join(" ");
}
if (query.type === "simple") {
return query.id + (query.id ? ": " : "") + (query.operator || "=") + ' "' +
query.value + '"';
}
throw new TypeError("This object is not a query");
}
_export("objectToSearchText", objectToSearchText);
src/queries/simplequery.js 0 → 100644
View file @ 4a6e56c2
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global newClass: true, Query: true,
query_class_dict: true, _export: true */
/**
* The SimpleQuery inherits from Query, and compares one metadata value
*
* @class SimpleQuery
* @extends Query
* @param {Object} [spec={}] The specifications
* @param {String} [spec.operator="="] The compare method to use
* @param {String} spec.key The metadata key
* @param {String} spec.value The value of the metadata to compare
*/
var SimpleQuery = newClass(Query, function (spec) {
/**
* Operator to use to compare object values
*
* @attribute operator
* @type String
* @default "="
* @optional
*/
this.operator = spec.operator || "=";
/**
* Key of the object which refers to the value to compare
*
* @attribute key
* @type String
*/
this.key = spec.key;
/**
* Value is used to do the comparison with the object value
*
* @attribute value
* @type String
*/
this.value = spec.value;
/**
* #crossLink "Query/match:method"
*/
this.match = function (item, wildcard_character) {
return this[this.operator](item[this.key], this.value, wildcard_character);
};
/**
* #crossLink "Query/toString:method"
*/
this.toString = function () {
return (this.key ? this.key + ": " : "") + (this.operator || "=") + ' "' +
this.value + '"';
};
/**
* #crossLink "Query/serialized:method"
*/
this.serialized = function () {
return {
"type": "simple",
"operator": this.operator,
"key": this.key,
"value": this.value
};
};
/**
* Comparison operator, test if this query value matches the item value
*
* @method =
* @param {String} object_value The value to compare
* @param {String} comparison_value The comparison value
* @param {String} wildcard_character The wildcard_character
* @return {Boolean} true if match, false otherwise
*/
this["="] = function (object_value, comparison_value,
wildcard_character) {
return Query.convertStringToRegExp(
comparison_value.toString(),
wildcard_character || "%"
).test(object_value.toString());
};
/**
* Comparison operator, test if this query value does not match the item value
*
* @method !=
* @param {String} object_value The value to compare
* @param {String} comparison_value The comparison value
* @param {String} wildcard_character The wildcard_character
* @return {Boolean} true if not match, false otherwise
*/
this["!="] = function (object_value, comparison_value,
wildcard_character) {
return !Query.convertStringTextToRegExp(
comparison_value.toString(),
wildcard_character || "%"
).test(object_value.toString());
};
/**
* Comparison operator, test if this query value is lower than the item value
*
* @method <
* @param {Number, String} object_value The value to compare
* @param {Number, String} comparison_value The comparison value
* @return {Boolean} true if lower, false otherwise
*/
this["<"] = function (object_value, comparison_value) {
return object_value < comparison_value;
};
/**
* Comparison operator, test if this query value is equal or lower than the
* item value
*
* @method <=
* @param {Number, String} object_value The value to compare
* @param {Number, String} comparison_value The comparison value
* @return {Boolean} true if equal or lower, false otherwise
*/
this["<="] = function (object_value, comparison_value) {
return object_value <= comparison_value;
};
/**
* Comparison operator, test if this query value is greater than the item
* value
*
* @method >
* @param {Number, String} object_value The value to compare
* @param {Number, String} comparison_value The comparison value
* @return {Boolean} true if greater, false otherwise
*/
this[">"] = function (object_value, comparison_value) {
return object_value > comparison_value;
};
/**
* Comparison operator, test if this query value is equal or greater than the
* item value
*
* @method >=
* @param {Number, String} object_value The value to compare
* @param {Number, String} comparison_value The comparison value
* @return {Boolean} true if equal or greater, false otherwise
*/
this[">="] = function (object_value, comparison_value) {
return object_value >= comparison_value;
};
});
query_class_dict.simple = SimpleQuery;
_export("SimpleQuery", SimpleQuery);
src/queries/tool.js 0 → 100644
View file @ 4a6e56c2
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global _export: true */
/**
* Create a class, manage inheritance, static methods,
* protected attributes and can hide methods or/and secure methods
*
* @param {Class} Class Classes to inherit from (0..n). The last class
* parameter will inherit from the previous one, and so on
* @param {Object} option Class option (0..n)
* @param {Boolean} [option.secure_methods=false] Make methods not configurable
* and not writable
* @param {Boolean} [option.hide_methods=false] Make methods not enumerable
* @param {Boolean} [option.secure_static_methods=true] Make static methods not
* configurable and not
* writable
* @param {Boolean} [option.hide_static_methods=false] Make static methods not
* enumerable
* @param {Object} [option.static_methods={}] Object of static methods
* @param {Function} constructor The new class constructor
* @return {Class} The new class
*/
function newClass() {
var j, k, constructors = [], option, new_class;
for (j = 0; j < arguments.length; j += 1) {
if (typeof arguments[j] === "function") {
constructors.push(arguments[j]);
} else if (typeof arguments[j] === "object") {
option = option || {};
for (k in arguments[j]) {
if (arguments[j].hasOwnProperty(k)) {
option[k] = arguments[j][k];
}
}
}
}
function postObjectCreation(that) {
// modify the object according to 'option'
var key;
if (option) {
for (key in that) {
if (that.hasOwnProperty(key)) {
if (typeof that[key] === "function") {
Object.defineProperty(that, key, {
"configurable": option.secure_methods ? false : true,
"enumerable": option.hide_methods ? false : true,
"writable": option.secure_methods ? false : true,
"value": that[key]
});
}
}
}
}
}
function postClassCreation(that) {
// modify the object according to 'option'
var key;
if (option) {
for (key in that) {
if (that.hasOwnProperty(key)) {
if (typeof that[key] === "function") {
Object.defineProperty(that, key, {
"configurable": option.secure_static_methods ===
false ? true : false,
"enumerable": option.hide_static_methods ? false : true,
"writable": option.secure_static_methods === false ? true : false,
"value": that[key]
});
}
}
}
}
}
new_class = function (spec, my) {
var i;
spec = spec || {};
my = my || {};
// don't use forEach !
for (i = 0; i < constructors.length; i += 1) {
constructors[i].apply(this, [spec, my]);
}
postObjectCreation(this);
return this;
};
option = option || {};
option.static_methods = option.static_methods || {};
for (j in option.static_methods) {
if (option.static_methods.hasOwnProperty(j)) {
new_class[j] = option.static_methods[j];
}
}
postClassCreation(new_class);
return new_class;
}
/**
* Escapes regexp special chars from a string.
*
* @param {String} string The string to escape
* @return {String} The escaped string
*/
function stringEscapeRegexpCharacters(string) {
if (typeof string === "string") {
return string.replace(/([\\\.\$\[\]\(\)\{\}\^\?\*\+\-])/g, "\\$1");
}
}
_export("stringEscapeRegexpCharacters", stringEscapeRegexpCharacters);
/**
* A sort function to sort items by key
*
* @param {String} key The key to sort on
* @param {String} [way="ascending"] 'ascending' or 'descending'
* @return {Function} The sort function
*/
function sortFunction(key, way) {
if (way === 'descending') {
return function (a, b) {
return a[key] < b[key] ? 1 : a[key] > b[key] ? -1 : 0;
};
}
return function (a, b) {
return a[key] > b[key] ? 1 : a[key] < b[key] ? -1 : 0;
};
}
test/jiotests.js
View file @ 4a6e56c2
......@@ -1174,7 +1174,7 @@ test ("AllDocs", function(){
// include docs
o.allDocsResponse = {};
o.allDocsResponse.rows = [];
o.allDocsResponse.total_rows = 15;
o.allDocsResponse.total_rows = m;
for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({
"id": "doc_"+(i < 10 ? "0"+i : i),
......@@ -1186,7 +1186,7 @@ test ("AllDocs", function(){
// alldocs
o.spy(o, "value", o.allDocsResponse, "All docs (include docs)");
o.jio.allDocs({"include_docs":true}, function (err, response) {
o.jio.allDocs({"include_docs": true}, function (err, response) {
if (response && response.rows) {
response.rows.sort(function (a, b) {
return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
......@@ -1197,65 +1197,56 @@ test ("AllDocs", function(){
o.tick(o);
// complex queries
o.thisShouldBeTheAnswer4 = [
{"title": "Inception", "year": 2010},
{"title": "The Dark Knight", "year": 2008},
{"title": "Lord of the Rings - Return of the King", "year": 2003},
{"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001},
{"title": "Fight Club", "year": 1999}
];
o.thisShouldBeTheAnswer4 = {"total_rows": 0, "rows": []};
o.allDocsResponse.rows.forEach(function (row) {
var new_row;
if (row.doc.year >= 1980) {
new_row = JSON.parse(JSON.stringify(row));
new_row.value.title = row.doc.title;
new_row.value.year = row.doc.year;
delete new_row.doc;
o.thisShouldBeTheAnswer4.rows.push(new_row);
o.thisShouldBeTheAnswer4.total_rows += 1;
}
});
o.thisShouldBeTheAnswer4.rows.sort(function (a, b) {
return a.value.year > b.value.year ? -1 :
a.value.year < b.value.year ? 1 : 0;
});
o.thisShouldBeTheAnswer4.total_rows = 5;
o.thisShouldBeTheAnswer4.rows.length = 5;
o.spy(o, "value", o.thisShouldBeTheAnswer4,
"allDocs (complex queries year >= 1980, all query options)");
o.jio.allDocs({
"query":{
"query":'(year: >= "1980")',
"filter": {
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['title','year']
},
"wildcard_character":'%'
}
"query": '(year: >= "1980")',
"limit": [0,5],
"sort_on": [["year", "descending"]],
"select_list": ["title", "year"]
}, o.f);
o.tick(o);
// empty query returns all
o.thisShouldBeTheAnswer5 = [
{"title": "The Good, The Bad and The Ugly"},
{"title": "The Dark Knight"},
{"title": "Star Wars Episode V"},
{"title": "Shawshank Redemption"},
{"title": "Schindlers List"},
{"title": "Pulp Fiction"},
{"title": "One flew over the Cuckoo's Nest"},
{"title": "Lord of the Rings - Return of the King"},
{"title": "Lord Of the Rings - Fellowship of the Ring"},
{"title": "Inception"},
{"title": "Godfellas"},
{"title": "Godfather 2"},
{"title": "Godfather"},
{"title": "Fight Club"},
{"title": "12 Angry Men"}
];
o.thisShouldBeTheAnswer5 = {"total_rows": 0, "rows": []};
o.allDocsResponse.rows.forEach(function (row) {
var new_row = JSON.parse(JSON.stringify(row));
new_row.value.title = row.doc.title;
o.thisShouldBeTheAnswer5.rows.push(new_row);
o.thisShouldBeTheAnswer5.total_rows += 1;
});
o.thisShouldBeTheAnswer5.rows.sort(function (a, b) {
return a.value.title > b.value.title ? -1 :
a.value.title < b.value.title ? 1 : 0;
});
o.spy(o, "value", o.thisShouldBeTheAnswer5,
"allDocs (empty query in complex query)");
o.jio.allDocs({
"query":{
"filter": {
"sort_on":[['title','descending']],
"select_list":['title']
},
"wildcard_character":'%'
}
}, function (err, response) {
if (response && response.rows) {
response.rows.sort(function (a, b) {
return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
});
}
o.f(err, response);
});
"sort_on": [["title", "descending"]],
"select_list": ["title"],
"include_docs": true
}, o.f);
o.tick(o);
o.jio.stop();
......@@ -4334,56 +4325,62 @@ test ("Post", function () {
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["findMeA"]},
{"name":"indexAB", "fields":["findMeA","findMeB"]}
],
"field_types": {
"findMeA": "string",
"findMeB": "string"
},
"sub_storage": {
"type": "local",
"username": "ipost",
"application_name": "ipost"
}
"type": "indexed",
"indices": [
{"id": "A", "index": ["title"]},
{"id": "B", "index": ["title", "year"]}
],
"sub_storage": {
"type": "local",
"username": "ipost",
"application_name": "ipost"
}
});
// post without id
o.spy (o, "status", undefined, "Post without id");
o.jio.post({}, o.f);
o.spy (o, "jobstatus", "done", "Post without id");
o.jio.post({}, function (err, response) {
o.id = (response || {}).id;
o.f(err, response);
});
o.tick(o);
// post non empty document
o.doc = {"_id": "some_id", "title": "myPost1",
"findMeA":"keyword_abc", "findMeB":"keyword_def"
};
o.doc = {"_id": "some_id", "title": "My Title",
"year": 2000, "hey": "def"};
o.spy (o, "value", {"ok": true, "id": "some_id"}, "Post document");
o.jio.post(o.doc, o.f);
o.tick(o);
// check document
o.fakeIndex = {
"_id": "ipost_indices.json",
"indexAB": {
"findMeA": {
"keyword_abc":["some_id"]
},
"findMeB": {
"keyword_def":["some_id"]
}
o.fakeIndexA = {
"_id": "A",
"indexing": ["title"],
"free": [],
"location": {
"some_id": 0
},
"indexA": {
"findMeA": {
"keyword_abc":["some_id"]
}
}
"database": [
{"_id": "some_id", "title": "My Title"}
]
};
o.jio.get({"_id": "ipost_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB = {
"_id": "B",
"indexing": ["title", "year"],
"free": [],
"location": {
"some_id": 0
},
"database": [
{"_id": "some_id", "title": "My Title", "year": 2000}
]
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// post with escapable characters
......@@ -4411,21 +4408,17 @@ test ("Put", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iput",
"application_name": "iput"
}
});
"type": "indexed",
"indices": [
{"id": "A", "index": ["author"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "iput",
"application_name": "iput"
}
});
// put without id
// error 20 -> document id required
......@@ -4434,151 +4427,241 @@ test ("Put", function(){
o.tick(o);
// put non empty document
o.doc = {"_id": "put1", "title": "myPut1", "author":"John Doe"};
o.doc = {"_id": "put1", "title": "myPut1", "author": "John Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"}, "Put-create document");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {
"author": {
"John Doe": ["put1"]
}
},
"indexAB": {
"author": {
"John Doe": ["put1"]
},
"year": {}
o.fakeIndexA = {
"_id": "A",
"indexing": ["author"],
"free": [],
"location": {
"put1": 0
},
"_id": "iput_indices.json"
"database": [{"_id": "put1", "author": "John Doe"}]
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [],
"location": {},
"database": []
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// modify document - modify keyword on index!
o.doc = {"_id": "put1", "title": "myPuttter1", "author":"Jane Doe"};
o.doc = {"_id": "put1", "title": "myPuttter1", "author": "Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"}, "Modify existing document");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {
"author": {
"Jane Doe": ["put1"]
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1"]
},
"year": {}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.fakeIndexA.database[0].author = "Jane Doe";
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
// add new document with same keyword!
o.doc = {"_id": "new_doc", "title": "myPut2", "author":"Jane Doe"};
o.doc = {"_id": "new_doc", "title": "myPut2", "author": "Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "new_doc"},
"Add new document with same keyword");
"Add new document with same keyword");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {
"author": {
"Jane Doe": ["put1", "new_doc"]
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1", "new_doc"]
},
"year": {}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.fakeIndexA.location.new_doc = 1;
o.fakeIndexA.database.push({"_id": "new_doc", "author": "Jane Doe"});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
// add second keyword to index file
o.doc = {"_id": "put1", "title": "myPut2", "author":"Jane Doe",
"year":"1912"};
o.doc = {"_id": "put1", "title": "myPut2", "author": "Jane Doe",
"year":"1912"};
o.spy (o, "value", {"ok": true, "id": "put1"},
"add second keyword to index file");
"add second keyword to index file");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {
"author": {
"Jane Doe": ["put1"]
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1"]
},
"year": {
"1912": ["put1"]
}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB.location.put1 = 0;
o.fakeIndexB.database.push({"_id": "put1", "year": "1912"});
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// remove a keyword from an existing document
o.doc = {"_id": "new_doc", "title": "myPut2"};
o.spy (o, "value", {"ok": true, "id": "new_doc"},
"Remove keyword from existing document");
"Remove keyword from existing document");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {
"author": {
"Jane Doe": ["put1"]
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1"]
},
"year": {
"1912": ["put1"]
}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
delete o.fakeIndexA.location.new_doc;
o.fakeIndexA.database[1] = null;
o.fakeIndexA.free.push(1);
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.jio.stop();
});
test("Check & Repair", function () {
var o = generateTools(this), i;
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"id": "A", "index": ["director"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "indexstoragerepair"
}
});
o.fakeIndexA = {
"_id": "A",
"indexing": ["director"],
"free": [],
"database": []
};
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [],
"database": []
};
for (i = 0; i < 10; i += 1) {
o.jio.put({
"_id": "id" + i,
"director": "D" + i,
"year": i,
"title": "T" + i
});
o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
o.fakeIndexA.database[o.tmp] = {"_id": "id" + i, "director": "D" + i};
o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
o.fakeIndexB.database[o.tmp] = {"_id": "id" + i, "year": i};
}
o.clock.tick(5000);
o.spy(o, "status", 40, "Check database");
o.jio.check({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "status", 40, "Check database");
o.jio.check({"_id": "B"}, o.f);
o.tick(o);
o.spy(o, "value", {"_id": "A", "ok": true}, "Repair database");
o.jio.repair({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "value", {"_id": "B", "ok": true}, "Repair database");
o.jio.repair({"_id": "B"}, o.f);
o.tick(o);
o.spy(o, "value", {"_id": "A", "ok": true}, "Check database again");
o.jio.check({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "value", {"_id": "B", "ok": true}, "Check database again");
o.jio.check({"_id": "B"}, o.f);
o.tick(o);
// check index file
o.spy(o, "value", o.fakeIndexA, "Manually check index file");
o.jio.get({"_id": "A"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Manually check index file");
o.jio.get({"_id": "B"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.jio2 = JIO.newJio({"type": "local", "username": "indexstoragerepair"});
o.jio2.put({"_id": "blah", "title": "t", "year": "y", "director": "d"});
o.clock.tick(1000);
o.jio2.stop();
o.fakeIndexA.database.unshift({"_id": "blah", "director": "d"});
o.fakeIndexB.database.unshift({"_id": "blah", "year": "y"});
o.spy(o, "status", 40, "Check Document");
o.jio.check({"_id": "blah"}, o.f)
o.tick(o);
o.spy(o, "value", {"id": "blah", "ok": true}, "Repair Document");
o.jio.repair({"_id": "blah"}, o.f)
o.tick(o);
o.spy(o, "value", {"id": "blah", "ok": true}, "Check Document again");
o.jio.repair({"_id": "blah"}, o.f)
o.tick(o);
// check index file
o.spy(o, "value", o.fakeIndexA, "Manually check index file");
o.jio.get({"_id": "A"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Manually check index file");
o.jio.get({"_id": "B"}, function (err, response) {
if (response) {
delete response.location;
response.database.sort(function (a, b) {
return a._id < b._id ? -1 : a._id > b._id ? 1 : 0;
});
}
o.f(err, response);
});
o.tick(o);
o.jio.stop();
});
test ("PutAttachment", function(){
// not sure these need to be run, because the index does not change
......@@ -4587,21 +4670,17 @@ test ("PutAttachment", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iputatt",
"application_name": "iputatt"
}
});
"type": "indexed",
"indices": [
{"id": "A", "index": ["author"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "iputatt",
"application_name": "iputatt"
}
});
// putAttachment without doc id
// error 20 -> document id required
......@@ -4624,7 +4703,7 @@ test ("PutAttachment", function(){
// putAttachment with document
o.doc = {"_id": "putattmt1","title": "myPutAttmt1"};
o.spy (o, "value", {"ok": true, "id": "putattmt1"},
"Put underlying document");
"Put underlying document");
o.jio.put(o.doc, o.f);
o.tick(o);
......@@ -4704,21 +4783,17 @@ test ("Get", function(){
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iget",
"application_name": "iget"
}
});
"type": "indexed",
"indices": [
{"id": "A", "index": ["author"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "iget",
"application_name": "iget"
}
});
// get inexistent document
o.spy(o, "status", 404, "Get inexistent document");
......@@ -4776,13 +4851,9 @@ test ("Remove", function(){
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
{"id": "A", "index": ["author"]},
{"id": "B", "index": ["year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "irem",
......@@ -4818,26 +4889,30 @@ test ("Remove", function(){
o.tick(o);
// check index
o.fakeIndex = {
"_id": "irem_indices.json",
"indexA": {
"author": {
"Martin Mustermann": ["removeAlso"]
}
},
"indexAB": {
"year": {
"2525": ["removeAlso"]
},
"author": {
"Martin Mustermann": ["removeAlso"]
}
}
o.fakeIndexA = {
"_id": "A",
"indexing": ["author"],
"free": [0],
"location": {
"removeAlso": 1
},
"database": [null, {"_id": "removeAlso", "author": "Martin Mustermann"}]
};
o.jio.get({"_id": "irem_indices.json"},function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [0],
"location": {
"removeAlso": 1
},
"database": [null, {"_id": "removeAlso", "year": "2525"}]
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// check document
......@@ -4877,29 +4952,18 @@ test ("Remove", function(){
o.tick(o);
// check index
o.fakeIndex = {
"_id": "irem_indices.json",
"indexA": {
"author":{
"Martin Mustermann": ["removeAlso"],
"Mrs Sunshine": ["remove3"]
}
},
"indexAB": {
"year": {
"1234": ["remove3"],
"2525": ["removeAlso"]
},
"author": {
"Martin Mustermann": ["removeAlso"],
"Mrs Sunshine": ["remove3"]
}
}
};
o.jio.get({"_id": "irem_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.fakeIndexA.free = [];
o.fakeIndexA.location.remove3 = 0;
o.fakeIndexA.database[0] = {"_id": "remove3", "author": "Mrs Sunshine"};
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB.free = [];
o.fakeIndexB.location.remove3 = 0;
o.fakeIndexB.database[0] = {"_id": "remove3", "year": "1234"};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// remove document and attachment together
......@@ -4909,26 +4973,18 @@ test ("Remove", function(){
o.tick(o);
// check index
o.fakeIndex = {
"_id": "irem_indices.json",
"indexA": {
"author": {
"Martin Mustermann": ["removeAlso"]
}
},
"indexAB": {
"year": {
"2525": ["removeAlso"]
},
"author": {
"Martin Mustermann": ["removeAlso"]
}
}
};
o.jio.get({"_id": "irem_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.fakeIndexA.free = [0];
delete o.fakeIndexA.location.remove3;
o.fakeIndexA.database[0] = null;
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.fakeIndexB.free = [0];
delete o.fakeIndexB.location.remove3;
o.fakeIndexB.database[0] = null;
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// check attachment
......@@ -4948,22 +5004,18 @@ test ("AllDocs", function () {
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iall",
"application_name": "iall"
}
});
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"id": "A", "index": ["author"]},
{"id": "B", "index": ["year"]}
],
"sub_storage": {
"type": "local",
"username": "iall",
"application_name": "iall"
}
});
// adding documents
o.all1 = { "_id": "dragon.doc",
......@@ -4992,34 +5044,25 @@ test ("AllDocs", function () {
o.tick(o);
// check index
o.fakeIndex = {
"_id": "iall_indices.json",
"indexA": {
"author": {
"Dr. No": ["dragon.doc"],
"Dr. Who": ["timemachine"],
"Dr. Snuggles": ["rocket.ppt"],
"Dr. House":["stick.jpg"]
}
o.fakeIndexA = {
"_id": "A",
"indexing": ["author"],
"free": [],
"location": {
"dragon.doc": 0,
"timemachine": 1,
"rocket.ppt": 2,
"stick.jpg": 3
},
"indexAB": {
"author": {
"Dr. No": ["dragon.doc"],
"Dr. Who": ["timemachine"],
"Dr. Snuggles": ["rocket.ppt"],
"Dr. House":["stick.jpg"]
},
"year": {
"1968": ["dragon.doc", "timemachine"],
"1985": ["rocket.ppt"],
"2005":["stick.jpg"]
}
}
"database": [
{"_id": "dragon.doc", "author": "Dr. No"},
{"_id": "timemachine", "author": "Dr. Who"},
{"_id": "rocket.ppt", "author": "Dr. Snuggles"},
{"_id": "stick.jpg", "author": "Dr. House"}
]
};
o.jio.get({"_id": "iall_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.thisShouldBeTheAnswer = {
......@@ -5035,19 +5078,6 @@ test ("AllDocs", function () {
o.jio.allDocs(o.f);
o.tick(o);
o.thisShouldBeTheAnswer2 = {
"rows": [
{"id": "dragon.doc", "key": "dragon.doc", "value": {}, "doc": o.all1 },
{"id": "timemachine", "key": "timemachine", "value": {}, "doc": o.all2 },
{"id": "rocket.ppt", "key": "rocket.ppt", "value": {}, "doc": o.all3 },
{"id": "stick.jpg", "key": "stick.jpg", "value": {}, "doc": o.all4 }
],
"total_rows": 4
}
o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
o.jio.allDocs({"include_docs":true}, o.f);
o.tick(o);
o.jio.stop();
});
......@@ -5058,16 +5088,9 @@ test ("AllDocs Complex Queries", function () {
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["director"]},
{"name":"indexAB", "fields":["title","year"]}
//,
//{"name":"indexABC", "fields":["title","year","director"]}
{"id":"A", "index": ["director"]},
{"id":"B", "index": ["title", "year"]}
],
"field_types": {
"director": "string",
"title": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "icomplex",
......@@ -5085,148 +5108,153 @@ test ("AllDocs Complex Queries", function () {
"One flew over the Cuckoo's Nest", "Inception", "Godfellas"
];
o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
1975,2010,1990
];
1975,2010,1990];
o.director = ["Frank Darabont", "Francis Ford Coppola",
"Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
"Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese"
]
];
o.fakeIndexA = {
"_id": "A",
"indexing": ["director"],
"free": [],
"location": {},
"database": []
};
o.fakeIndexB = {
"_id": "B",
"indexing": ["title", "year"],
"free": [],
"location": {},
"database": []
};
for (i = 0; i < m; i += 1) {
o.fakeDoc = {};
o.fakeDoc._id = ""+i;
o.fakeDoc.title = o.titles[i];
o.fakeDoc.year = o.years[i];
o.fakeDoc.director = o.director[i];
o.jio.put(o.fakeDoc);
o.jio.put({
"_id": "" + i,
"director": o.director[i],
"year": o.years[i],
"title": o.titles[i]
});
o.tmp = o.fakeIndexA.free.pop() || o.fakeIndexA.database.length;
o.fakeIndexA.database[o.tmp] = {"_id": "" + i, "director": o.director[i]};
o.fakeIndexA.location["" + i] = o.tmp;
o.tmp = o.fakeIndexB.free.pop() || o.fakeIndexB.database.length;
o.fakeIndexB.database[o.tmp] = {
"_id": "" + i,
"year": o.years[i],
"title": o.titles[i]
};
o.fakeIndexB.location["" + i] = o.tmp;
o.clock.tick(1000);
}
// o.clock.tick(1000);
// check index file
o.spy(o, "value", o.fakeIndexA, "Check index file");
o.jio.get({"_id": "A"}, o.f);
o.tick(o);
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o);
// response
o.allDocsResponse = {};
o.allDocsResponse.rows = [];
o.allDocsResponse.total_rows = 15;
o.allDocsResponse.total_rows = m;
for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({
"id": ""+i,
"key": ""+i,
"value": {}
});
};
// alldocs
o.jio.allDocs(function (e, r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse,
"AllDocs response generated from index"
);
});
o.clock.tick(1000);
// include docs
o.allDocsResponse2 = {};
o.allDocsResponse2.rows = [];
o.allDocsResponse2.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse2.rows.push({
"id": ""+i,
"key": ""+i,
"value": {},
"doc": localstorage.getItem(o.localpath+"/"+i)
"doc": {
"_id": ""+i,
"title": o.titles[i],
"year": o.years[i],
"director": o.director[i]
}
});
};
}
o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
for (i = 0; i < o.response.rows.length; i += 1) {
delete o.response.rows[i].doc;
}
// alldocs
o.jio.allDocs({"include_docs":true}, function(e,r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse2,
"AllDocs response generated from index (include docs)"
);
});
o.clock.tick(1000);
o.spy(o, "value", o.response, "AllDocs response generated from index");
o.jio.allDocs(o.f);
o.tick(o, 1000);
// complex queries
o.thisShouldBeTheAnswer4 = [
{"title": "Inception", "year": 2010},
{"title": "The Dark Knight", "year": 2008},
{"title": "Lord of the Rings - Return of the King", "year": 2003},
{"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001},
{"title": "Fight Club", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer4,
o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
i = 0;
while (i < o.response.rows.length) {
if (o.response.rows[i].year < 1980) {
o.response.rows.splice(i, 1);
} else {
o.response.rows[i].value = {
"year": o.response.rows[i].doc.year,
"title": o.response.rows[i].doc.title
}
delete o.response.rows[i].doc;
i += 1;
}
}
o.response.rows.sort(function (a, b) {
return a.value.year > b.value.year ? -1 :
a.value.year < b.value.year ? 1 : 0;
});
o.response.rows.length = 5;
o.response.total_rows = 5;
o.spy(o, "value", o.response,
"allDocs (complex queries year >= 1980, index used to do query)");
o.jio.allDocs({
"query":{
// "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": {
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['title','year']
},
"wildcard_character":'%'
}
// "query":'(year: >= "1980" AND year: < "2000")',
"query": '(year: >= "1980")',
"limit": [0, 5],
"sort_on": [['year', 'descending']],
"select_list": ['title', 'year']
}, o.f);
o.tick(o);
// complex queries
o.thisShouldBeTheAnswer5 = [
{"director": "Christopher Nolan", "year": 2010},
{"director": "Christopher Nolan", "year": 2008},
{"director": "Peter Jackson", "year": 2003},
{"director": "Peter Jackson", "year": 2001},
{"director": "David Fincher", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer5,
o.spy(o, "value", {"total_rows": 0, "rows": []},
"allDocs (complex queries year >= 1980, can't use index)");
o.jio.allDocs({
"query":{
// "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": {
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['director','year']
},
"wildcard_character":'%'
}
// "query":'(year: >= "1980" AND year: < "2000")',
"query": '(year: >= "1980")',
"limit": [0, 5],
"sort_on": [['year','descending']],
"select_list": ['director', 'year']
}, o.f);
o.tick(o);
// empty query returns all
o.thisShouldBeTheAnswer6 = [
{"title": "The Good, The Bad and The Ugly"},
{"title": "The Dark Knight"},
{"title": "Star Wars Episode V"},
{"title": "Shawshank Redemption"},
{"title": "Schindlers List"},
{"title": "Pulp Fiction"},
{"title": "One flew over the Cuckoo's Nest"},
{"title": "Lord of the Rings - Return of the King"},
{"title": "Lord Of the Rings - Fellowship of the Ring"},
{"title": "Inception"},
{"title": "Godfellas"},
{"title": "Godfather 2"},
{"title": "Godfather"},
{"title": "Fight Club"},
{"title": "12 Angry Men"}
];
o.spy(o, "value", o.thisShouldBeTheAnswer6,
o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
i = 0;
while (i < o.response.rows.length) {
o.response.rows[i].value.title =
o.response.rows[i].doc.title;
delete o.response.rows[i].doc;
i += 1;
}
o.response.rows.sort(function (a, b) {
return a.value.title > b.value.title ? -1 :
a.value.title < b.value.title ? 1 : 0;
});
o.spy(o, "value", o.response,
"allDocs (empty query in complex query)");
o.jio.allDocs({
"query":{
"filter": {
"sort_on":[['title','descending']],
"select_list":['title']
},
"wildcard_character":'%'
}
"sort_on":[['title','descending']],
"select_list":['title']
}, o.f);
o.tick(o);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7