Commit 53887d79 by Julien Muchembled

mysql: new configuration parameter to choose engine (InnoDB or TokuDB)

1 parent 2a3392eb
......@@ -48,6 +48,7 @@ partitions: 12
# - MySQL: [user[:password]@]database[unix_socket]
# Database must be created manually.
# - SQLite: path
# engine: Optionnal parameter for MySQL. Can be InnoDB (default) or TokuDB.
# Admin node
[admin]
......
......@@ -61,6 +61,9 @@ class ConfigurationManager(object):
def getDatabase(self):
return self.__get('database')
def getEngine(self):
return self.__get('engine', True)
def getWait(self):
# BUG
return self.__get('wait')
......
......@@ -36,6 +36,7 @@ parser.add_option('-c', '--cluster', help = 'the cluster name')
parser.add_option('-m', '--masters', help = 'master node list')
parser.add_option('-a', '--adapter', help = 'database adapter to use')
parser.add_option('-d', '--database', help = 'database connections string')
parser.add_option('-e', '--engine', help = 'database engine')
parser.add_option('-D', '--dynamic-master-list', help='path of the file '
'containing dynamic master node list')
parser.add_option('-w', '--wait', help='seconds to wait for backend to be '
......@@ -59,6 +60,7 @@ def main(args=None):
cluster = options.cluster,
masters = options.masters,
database = options.database,
engine = options.engine,
reset = options.reset,
adapter = options.adapter,
wait = options.wait,
......
......@@ -50,7 +50,7 @@ class Application(object):
self.nm = NodeManager(config.getDynamicMasterList())
self.tm = TransactionManager(self)
self.dm = buildDatabaseManager(config.getAdapter(),
(config.getDatabase(), config.getWait())
(config.getDatabase(), config.getEngine(), config.getWait()),
)
# load master nodes
......
......@@ -294,7 +294,7 @@ class ImporterDatabaseManager(DatabaseManager):
self.zodb = ((x, dict(config.items(x))) for x in sections)
self.compress = main.get('compress', 1)
self.db = buildDatabaseManager(main['adapter'],
(main['database'], main['wait']))
(main['database'], main.get('engine'), main['wait']))
for x in """query erase getConfiguration _setConfiguration
getPartitionTable changePartitionTable getUnfinishedTIDList
dropUnfinishedData storeTransaction finishTransaction
......
......@@ -42,10 +42,17 @@ class CreationUndone(Exception):
class DatabaseManager(object):
"""This class only describes an interface for database managers."""
def __init__(self, database, wait=0):
ENGINES = ()
def __init__(self, database, engine=None, wait=0):
"""
Initialize the object.
"""
if engine:
if engine not in self.ENGINES:
raise ValueError("Unsupported engine: %r not in %r"
% (engine, self.ENGINES))
self._engine = engine
self._wait = wait
self._parse(database)
......
......@@ -44,8 +44,8 @@ def splitOIDField(tid, oids):
class MySQLDatabaseManager(DatabaseManager):
"""This class manages a database on MySQL."""
# WARNING: some parts are not concurrent safe (ex: holdData)
# (there must be only 1 writable connection per DB)
ENGINES = "InnoDB", "TokuDB"
_engine = ENGINES[0] # default engine
# Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because
# 'select count(*) from obj' sometimes returns incorrect values
......@@ -142,12 +142,13 @@ class MySQLDatabaseManager(DatabaseManager):
def _setup(self):
self._config.clear()
q = self.query
p = engine = self._engine
# The table "config" stores configuration parameters which affect the
# persistent data.
q("""CREATE TABLE IF NOT EXISTS config (
name VARBINARY(255) NOT NULL PRIMARY KEY,
value VARBINARY(255) NULL
) ENGINE = InnoDB""")
) ENGINE=""" + engine)
# The table "pt" stores a partition table.
q("""CREATE TABLE IF NOT EXISTS pt (
......@@ -155,10 +156,11 @@ class MySQLDatabaseManager(DatabaseManager):
nid INT NOT NULL,
state TINYINT UNSIGNED NOT NULL,
PRIMARY KEY (rid, nid)
) ENGINE = InnoDB""")
) ENGINE=""" + engine)
p = self._use_partition and """ PARTITION BY LIST (`partition`) (
PARTITION dummy VALUES IN (NULL))""" or ''
if self._use_partition:
p += """ PARTITION BY LIST (`partition`) (
PARTITION dummy VALUES IN (NULL))"""
# The table "trans" stores information on committed transactions.
q("""CREATE TABLE IF NOT EXISTS trans (
......@@ -171,7 +173,7 @@ class MySQLDatabaseManager(DatabaseManager):
ext BLOB NOT NULL,
ttid BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (`partition`, tid)
) ENGINE = InnoDB""" + p)
) ENGINE=""" + p)
# The table "obj" stores committed object metadata.
q("""CREATE TABLE IF NOT EXISTS obj (
......@@ -183,7 +185,10 @@ class MySQLDatabaseManager(DatabaseManager):
PRIMARY KEY (`partition`, tid, oid),
KEY (`partition`, oid, tid),
KEY (data_id)
) ENGINE = InnoDB""" + p)
) ENGINE=""" + p)
if engine == "TokuDB":
engine += " compression='tokudb_uncompressed'"
# The table "data" stores object data.
# We'd like to have partial index on 'hash' colum (e.g. hash(4))
......@@ -193,7 +198,7 @@ class MySQLDatabaseManager(DatabaseManager):
hash BINARY(20) NOT NULL UNIQUE,
compression TINYINT UNSIGNED NULL,
value LONGBLOB NULL
) ENGINE = InnoDB""")
) ENGINE=""" + engine)
# The table "ttrans" stores information on uncommitted transactions.
q("""CREATE TABLE IF NOT EXISTS ttrans (
......@@ -205,7 +210,7 @@ class MySQLDatabaseManager(DatabaseManager):
description BLOB NOT NULL,
ext BLOB NOT NULL,
ttid BIGINT UNSIGNED NOT NULL
) ENGINE = InnoDB""")
) ENGINE=""" + engine)
# The table "tobj" stores uncommitted object metadata.
q("""CREATE TABLE IF NOT EXISTS tobj (
......@@ -215,7 +220,7 @@ class MySQLDatabaseManager(DatabaseManager):
data_id BIGINT UNSIGNED NULL,
value_tid BIGINT UNSIGNED NULL,
PRIMARY KEY (tid, oid)
) ENGINE = InnoDB""")
) ENGINE=""" + engine)
self._uncommitted_data.update(q("SELECT data_id, count(*)"
" FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id"))
......
......@@ -24,24 +24,19 @@ from neo.storage.database.mysqldb import MySQLDatabaseManager
NEO_SQL_DATABASE = 'test_mysqldb0'
NEO_SQL_USER = 'test'
class StorageMySQSLdbTests(StorageDBTests):
class StorageMySQLdbTests(StorageDBTests):
engine = None
def getDB(self, reset=0):
self.prepareDatabase(number=1, prefix=NEO_SQL_DATABASE[:-1])
# db manager
database = '%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE)
db = MySQLDatabaseManager(database, 0)
db.setup(reset)
return db
def test_MySQLDatabaseManagerInit(self):
db = MySQLDatabaseManager('%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE),
0)
# init
db = MySQLDatabaseManager(database, self.engine)
self.assertEqual(db.db, NEO_SQL_DATABASE)
self.assertEqual(db.user, NEO_SQL_USER)
# & connect
self.assertTrue(isinstance(db.conn, MySQLdb.connection))
db.setup(reset)
return db
def test_query1(self):
# fake result object
......@@ -96,6 +91,10 @@ class StorageMySQSLdbTests(StorageDBTests):
self.assertEqual(self.db.escape('a"b'), 'a\\"b')
self.assertEqual(self.db.escape("a'b"), "a\\'b")
class StorageMySQLdbTokuDBTests(StorageMySQLdbTests):
engine = "TokuDB"
del StorageDBTests
if __name__ == "__main__":
......
......@@ -21,7 +21,7 @@ from neo.storage.database.sqlite import SQLiteDatabaseManager
class StorageSQLiteTests(StorageDBTests):
def getDB(self, reset=0):
db = SQLiteDatabaseManager(':memory:', 0)
db = SQLiteDatabaseManager(':memory:')
db.setup(reset)
return db
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!