Commit 53887d79 authored by Julien Muchembled's avatar Julien Muchembled

mysql: new configuration parameter to choose engine (InnoDB or TokuDB)

parent 2a3392eb
...@@ -48,6 +48,7 @@ partitions: 12 ...@@ -48,6 +48,7 @@ partitions: 12
# - MySQL: [user[:password]@]database[unix_socket] # - MySQL: [user[:password]@]database[unix_socket]
# Database must be created manually. # Database must be created manually.
# - SQLite: path # - SQLite: path
# engine: Optionnal parameter for MySQL. Can be InnoDB (default) or TokuDB.
# Admin node # Admin node
[admin] [admin]
......
...@@ -61,6 +61,9 @@ class ConfigurationManager(object): ...@@ -61,6 +61,9 @@ class ConfigurationManager(object):
def getDatabase(self): def getDatabase(self):
return self.__get('database') return self.__get('database')
def getEngine(self):
return self.__get('engine', True)
def getWait(self): def getWait(self):
# BUG # BUG
return self.__get('wait') return self.__get('wait')
......
...@@ -36,6 +36,7 @@ parser.add_option('-c', '--cluster', help = 'the cluster name') ...@@ -36,6 +36,7 @@ parser.add_option('-c', '--cluster', help = 'the cluster name')
parser.add_option('-m', '--masters', help = 'master node list') parser.add_option('-m', '--masters', help = 'master node list')
parser.add_option('-a', '--adapter', help = 'database adapter to use') parser.add_option('-a', '--adapter', help = 'database adapter to use')
parser.add_option('-d', '--database', help = 'database connections string') parser.add_option('-d', '--database', help = 'database connections string')
parser.add_option('-e', '--engine', help = 'database engine')
parser.add_option('-D', '--dynamic-master-list', help='path of the file ' parser.add_option('-D', '--dynamic-master-list', help='path of the file '
'containing dynamic master node list') 'containing dynamic master node list')
parser.add_option('-w', '--wait', help='seconds to wait for backend to be ' parser.add_option('-w', '--wait', help='seconds to wait for backend to be '
...@@ -59,6 +60,7 @@ def main(args=None): ...@@ -59,6 +60,7 @@ def main(args=None):
cluster = options.cluster, cluster = options.cluster,
masters = options.masters, masters = options.masters,
database = options.database, database = options.database,
engine = options.engine,
reset = options.reset, reset = options.reset,
adapter = options.adapter, adapter = options.adapter,
wait = options.wait, wait = options.wait,
......
...@@ -50,7 +50,7 @@ class Application(object): ...@@ -50,7 +50,7 @@ class Application(object):
self.nm = NodeManager(config.getDynamicMasterList()) self.nm = NodeManager(config.getDynamicMasterList())
self.tm = TransactionManager(self) self.tm = TransactionManager(self)
self.dm = buildDatabaseManager(config.getAdapter(), self.dm = buildDatabaseManager(config.getAdapter(),
(config.getDatabase(), config.getWait()) (config.getDatabase(), config.getEngine(), config.getWait()),
) )
# load master nodes # load master nodes
......
...@@ -294,7 +294,7 @@ class ImporterDatabaseManager(DatabaseManager): ...@@ -294,7 +294,7 @@ class ImporterDatabaseManager(DatabaseManager):
self.zodb = ((x, dict(config.items(x))) for x in sections) self.zodb = ((x, dict(config.items(x))) for x in sections)
self.compress = main.get('compress', 1) self.compress = main.get('compress', 1)
self.db = buildDatabaseManager(main['adapter'], self.db = buildDatabaseManager(main['adapter'],
(main['database'], main['wait'])) (main['database'], main.get('engine'), main['wait']))
for x in """query erase getConfiguration _setConfiguration for x in """query erase getConfiguration _setConfiguration
getPartitionTable changePartitionTable getUnfinishedTIDList getPartitionTable changePartitionTable getUnfinishedTIDList
dropUnfinishedData storeTransaction finishTransaction dropUnfinishedData storeTransaction finishTransaction
......
...@@ -42,10 +42,17 @@ class CreationUndone(Exception): ...@@ -42,10 +42,17 @@ class CreationUndone(Exception):
class DatabaseManager(object): class DatabaseManager(object):
"""This class only describes an interface for database managers.""" """This class only describes an interface for database managers."""
def __init__(self, database, wait=0): ENGINES = ()
def __init__(self, database, engine=None, wait=0):
""" """
Initialize the object. Initialize the object.
""" """
if engine:
if engine not in self.ENGINES:
raise ValueError("Unsupported engine: %r not in %r"
% (engine, self.ENGINES))
self._engine = engine
self._wait = wait self._wait = wait
self._parse(database) self._parse(database)
......
...@@ -44,8 +44,8 @@ def splitOIDField(tid, oids): ...@@ -44,8 +44,8 @@ def splitOIDField(tid, oids):
class MySQLDatabaseManager(DatabaseManager): class MySQLDatabaseManager(DatabaseManager):
"""This class manages a database on MySQL.""" """This class manages a database on MySQL."""
# WARNING: some parts are not concurrent safe (ex: holdData) ENGINES = "InnoDB", "TokuDB"
# (there must be only 1 writable connection per DB) _engine = ENGINES[0] # default engine
# Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because # Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because
# 'select count(*) from obj' sometimes returns incorrect values # 'select count(*) from obj' sometimes returns incorrect values
...@@ -142,12 +142,13 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -142,12 +142,13 @@ class MySQLDatabaseManager(DatabaseManager):
def _setup(self): def _setup(self):
self._config.clear() self._config.clear()
q = self.query q = self.query
p = engine = self._engine
# The table "config" stores configuration parameters which affect the # The table "config" stores configuration parameters which affect the
# persistent data. # persistent data.
q("""CREATE TABLE IF NOT EXISTS config ( q("""CREATE TABLE IF NOT EXISTS config (
name VARBINARY(255) NOT NULL PRIMARY KEY, name VARBINARY(255) NOT NULL PRIMARY KEY,
value VARBINARY(255) NULL value VARBINARY(255) NULL
) ENGINE = InnoDB""") ) ENGINE=""" + engine)
# The table "pt" stores a partition table. # The table "pt" stores a partition table.
q("""CREATE TABLE IF NOT EXISTS pt ( q("""CREATE TABLE IF NOT EXISTS pt (
...@@ -155,10 +156,11 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -155,10 +156,11 @@ class MySQLDatabaseManager(DatabaseManager):
nid INT NOT NULL, nid INT NOT NULL,
state TINYINT UNSIGNED NOT NULL, state TINYINT UNSIGNED NOT NULL,
PRIMARY KEY (rid, nid) PRIMARY KEY (rid, nid)
) ENGINE = InnoDB""") ) ENGINE=""" + engine)
p = self._use_partition and """ PARTITION BY LIST (`partition`) ( if self._use_partition:
PARTITION dummy VALUES IN (NULL))""" or '' p += """ PARTITION BY LIST (`partition`) (
PARTITION dummy VALUES IN (NULL))"""
# The table "trans" stores information on committed transactions. # The table "trans" stores information on committed transactions.
q("""CREATE TABLE IF NOT EXISTS trans ( q("""CREATE TABLE IF NOT EXISTS trans (
...@@ -171,7 +173,7 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -171,7 +173,7 @@ class MySQLDatabaseManager(DatabaseManager):
ext BLOB NOT NULL, ext BLOB NOT NULL,
ttid BIGINT UNSIGNED NOT NULL, ttid BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (`partition`, tid) PRIMARY KEY (`partition`, tid)
) ENGINE = InnoDB""" + p) ) ENGINE=""" + p)
# The table "obj" stores committed object metadata. # The table "obj" stores committed object metadata.
q("""CREATE TABLE IF NOT EXISTS obj ( q("""CREATE TABLE IF NOT EXISTS obj (
...@@ -183,7 +185,10 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -183,7 +185,10 @@ class MySQLDatabaseManager(DatabaseManager):
PRIMARY KEY (`partition`, tid, oid), PRIMARY KEY (`partition`, tid, oid),
KEY (`partition`, oid, tid), KEY (`partition`, oid, tid),
KEY (data_id) KEY (data_id)
) ENGINE = InnoDB""" + p) ) ENGINE=""" + p)
if engine == "TokuDB":
engine += " compression='tokudb_uncompressed'"
# The table "data" stores object data. # The table "data" stores object data.
# We'd like to have partial index on 'hash' colum (e.g. hash(4)) # We'd like to have partial index on 'hash' colum (e.g. hash(4))
...@@ -193,7 +198,7 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -193,7 +198,7 @@ class MySQLDatabaseManager(DatabaseManager):
hash BINARY(20) NOT NULL UNIQUE, hash BINARY(20) NOT NULL UNIQUE,
compression TINYINT UNSIGNED NULL, compression TINYINT UNSIGNED NULL,
value LONGBLOB NULL value LONGBLOB NULL
) ENGINE = InnoDB""") ) ENGINE=""" + engine)
# The table "ttrans" stores information on uncommitted transactions. # The table "ttrans" stores information on uncommitted transactions.
q("""CREATE TABLE IF NOT EXISTS ttrans ( q("""CREATE TABLE IF NOT EXISTS ttrans (
...@@ -205,7 +210,7 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -205,7 +210,7 @@ class MySQLDatabaseManager(DatabaseManager):
description BLOB NOT NULL, description BLOB NOT NULL,
ext BLOB NOT NULL, ext BLOB NOT NULL,
ttid BIGINT UNSIGNED NOT NULL ttid BIGINT UNSIGNED NOT NULL
) ENGINE = InnoDB""") ) ENGINE=""" + engine)
# The table "tobj" stores uncommitted object metadata. # The table "tobj" stores uncommitted object metadata.
q("""CREATE TABLE IF NOT EXISTS tobj ( q("""CREATE TABLE IF NOT EXISTS tobj (
...@@ -215,7 +220,7 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -215,7 +220,7 @@ class MySQLDatabaseManager(DatabaseManager):
data_id BIGINT UNSIGNED NULL, data_id BIGINT UNSIGNED NULL,
value_tid BIGINT UNSIGNED NULL, value_tid BIGINT UNSIGNED NULL,
PRIMARY KEY (tid, oid) PRIMARY KEY (tid, oid)
) ENGINE = InnoDB""") ) ENGINE=""" + engine)
self._uncommitted_data.update(q("SELECT data_id, count(*)" self._uncommitted_data.update(q("SELECT data_id, count(*)"
" FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id")) " FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id"))
......
...@@ -24,24 +24,19 @@ from neo.storage.database.mysqldb import MySQLDatabaseManager ...@@ -24,24 +24,19 @@ from neo.storage.database.mysqldb import MySQLDatabaseManager
NEO_SQL_DATABASE = 'test_mysqldb0' NEO_SQL_DATABASE = 'test_mysqldb0'
NEO_SQL_USER = 'test' NEO_SQL_USER = 'test'
class StorageMySQSLdbTests(StorageDBTests): class StorageMySQLdbTests(StorageDBTests):
engine = None
def getDB(self, reset=0): def getDB(self, reset=0):
self.prepareDatabase(number=1, prefix=NEO_SQL_DATABASE[:-1]) self.prepareDatabase(number=1, prefix=NEO_SQL_DATABASE[:-1])
# db manager # db manager
database = '%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE) database = '%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE)
db = MySQLDatabaseManager(database, 0) db = MySQLDatabaseManager(database, self.engine)
db.setup(reset)
return db
def test_MySQLDatabaseManagerInit(self):
db = MySQLDatabaseManager('%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE),
0)
# init
self.assertEqual(db.db, NEO_SQL_DATABASE) self.assertEqual(db.db, NEO_SQL_DATABASE)
self.assertEqual(db.user, NEO_SQL_USER) self.assertEqual(db.user, NEO_SQL_USER)
# & connect db.setup(reset)
self.assertTrue(isinstance(db.conn, MySQLdb.connection)) return db
def test_query1(self): def test_query1(self):
# fake result object # fake result object
...@@ -96,6 +91,10 @@ class StorageMySQSLdbTests(StorageDBTests): ...@@ -96,6 +91,10 @@ class StorageMySQSLdbTests(StorageDBTests):
self.assertEqual(self.db.escape('a"b'), 'a\\"b') self.assertEqual(self.db.escape('a"b'), 'a\\"b')
self.assertEqual(self.db.escape("a'b"), "a\\'b") self.assertEqual(self.db.escape("a'b"), "a\\'b")
class StorageMySQLdbTokuDBTests(StorageMySQLdbTests):
engine = "TokuDB"
del StorageDBTests del StorageDBTests
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -21,7 +21,7 @@ from neo.storage.database.sqlite import SQLiteDatabaseManager ...@@ -21,7 +21,7 @@ from neo.storage.database.sqlite import SQLiteDatabaseManager
class StorageSQLiteTests(StorageDBTests): class StorageSQLiteTests(StorageDBTests):
def getDB(self, reset=0): def getDB(self, reset=0):
db = SQLiteDatabaseManager(':memory:', 0) db = SQLiteDatabaseManager(':memory:')
db.setup(reset) db.setup(reset)
return db return db
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment