Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
d289050e
Commit
d289050e
authored
Jan 05, 2018
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Plain Diff
storage: automatic upgrade of 'obj' table (change of indices)
parents
ca7acefc
3c7a3160
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
47 additions
and
23 deletions
+47
-23
neo/storage/database/mysqldb.py
neo/storage/database/mysqldb.py
+26
-16
neo/storage/database/sqlite.py
neo/storage/database/sqlite.py
+21
-7
No files found.
neo/storage/database/mysqldb.py
View file @
d289050e
...
...
@@ -50,7 +50,7 @@ def getPrintableQuery(query, max=70):
class
MySQLDatabaseManager
(
DatabaseManager
):
"""This class manages a database on MySQL."""
VERSION
=
1
VERSION
=
2
ENGINES
=
"InnoDB"
,
"RocksDB"
,
"TokuDB"
_engine
=
ENGINES
[
0
]
# default engine
...
...
@@ -181,6 +181,17 @@ class MySQLDatabaseManager(DatabaseManager):
self
.
_checkNoUnfinishedTransactions
()
self
.
query
(
"DROP TABLE IF EXISTS ttrans"
)
def
_migrate2
(
self
,
schema_dict
):
q
=
self
.
query
if
self
.
nonempty
(
'obj'
)
is
None
:
if
self
.
nonempty
(
'new_obj'
)
is
None
:
return
else
:
q
(
"DROP TABLE IF EXISTS new_obj"
)
q
(
schema_dict
.
pop
(
'obj'
)
%
'new_obj'
+
" SELECT * FROM obj"
)
q
(
"DROP TABLE obj"
)
q
(
"ALTER TABLE new_obj RENAME TO obj"
)
def
_setup
(
self
,
dedup
=
False
):
self
.
_config
.
clear
()
q
=
self
.
query
...
...
@@ -226,8 +237,8 @@ class MySQLDatabaseManager(DatabaseManager):
tid BIGINT UNSIGNED NOT NULL,
data_id BIGINT UNSIGNED NULL,
value_tid BIGINT UNSIGNED NULL,
PRIMARY KEY (`partition`,
tid, o
id),
KEY
(`partition`, oid, t
id),
PRIMARY KEY (`partition`,
oid, t
id),
KEY
tid (`partition`, tid, o
id),
KEY (data_id)
) ENGINE="""
+
p
...
...
@@ -341,15 +352,14 @@ class MySQLDatabaseManager(DatabaseManager):
offset_list
=
self
.
_getAssignedPartitionList
()
p64
=
util
.
p64
q
=
self
.
query
sql
=
(
"SELECT MAX(tid) FROM %s FORCE INDEX (PRIMARY)"
" WHERE `partition`=%s"
)
sql
=
"SELECT MAX(tid) FROM %s WHERE `partition`=%s"
trans
,
obj
=
({
partition
:
p64
(
tid
)
for
partition
in
offset_list
for
tid
,
in
q
(
sql
%
(
t
,
partition
))
if
tid
is
not
None
}
for
t
in
(
'trans
'
,
'obj
'
))
for
t
in
(
'trans
FORCE INDEX (PRIMARY)'
,
'obj FORCE INDEX (tid)
'
))
oid
=
self
.
_sqlmax
(
"SELECT MAX(oid) FROM obj FORCE INDEX (
`partition`
)"
"SELECT MAX(oid) FROM obj FORCE INDEX (
PRIMARY
)"
" WHERE `partition`=%s"
,
offset_list
)
return
trans
,
obj
,
None
if
oid
is
None
else
p64
(
oid
)
...
...
@@ -369,7 +379,7 @@ class MySQLDatabaseManager(DatabaseManager):
def
getLastObjectTID
(
self
,
oid
):
oid
=
util
.
u64
(
oid
)
r
=
self
.
query
(
"SELECT tid FROM obj FORCE INDEX(
`partition`
)"
r
=
self
.
query
(
"SELECT tid FROM obj FORCE INDEX(
PRIMARY
)"
" WHERE `partition`=%d AND oid=%d"
" ORDER BY tid DESC LIMIT 1"
%
(
self
.
_getReadablePartition
(
oid
),
oid
))
...
...
@@ -377,7 +387,7 @@ class MySQLDatabaseManager(DatabaseManager):
def
_getNextTID
(
self
,
*
args
):
# partition, oid, tid
r
=
self
.
query
(
"SELECT tid FROM obj"
" FORCE INDEX(
`partition`
)"
" FORCE INDEX(
PRIMARY
)"
" WHERE `partition`=%d AND oid=%d AND tid>%d"
" ORDER BY tid LIMIT 1"
%
args
)
return
r
[
0
][
0
]
if
r
else
None
...
...
@@ -386,7 +396,7 @@ class MySQLDatabaseManager(DatabaseManager):
q
=
self
.
query
partition
=
self
.
_getReadablePartition
(
oid
)
sql
=
(
'SELECT tid, compression, data.hash, value, value_tid'
' FROM obj FORCE INDEX(
`partition`
)'
' FROM obj FORCE INDEX(
PRIMARY
)'
' LEFT JOIN data ON (obj.data_id = data.id)'
' WHERE `partition` = %d AND oid = %d'
)
%
(
partition
,
oid
)
if
before_tid
is
not
None
:
...
...
@@ -443,7 +453,7 @@ class MySQLDatabaseManager(DatabaseManager):
for
partition
in
offset_list
:
where
=
" WHERE `partition`=%d"
%
partition
data_id_list
=
[
x
for
x
,
in
q
(
"SELECT DISTINCT data_id FROM obj FORCE INDEX(
PRIMARY
)"
q
(
"SELECT DISTINCT data_id FROM obj FORCE INDEX(
tid
)"
+
where
)
if
x
]
if
not
self
.
_use_partition
:
...
...
@@ -608,7 +618,7 @@ class MySQLDatabaseManager(DatabaseManager):
del
_structLL
def
_getDataTID
(
self
,
oid
,
tid
=
None
,
before_tid
=
None
):
sql
=
(
'SELECT tid, value_tid FROM obj FORCE INDEX(
`partition`
)'
sql
=
(
'SELECT tid, value_tid FROM obj FORCE INDEX(
PRIMARY
)'
' WHERE `partition` = %d AND oid = %d'
)
%
(
self
.
_getReadablePartition
(
oid
),
oid
)
if
tid
is
not
None
:
...
...
@@ -699,7 +709,7 @@ class MySQLDatabaseManager(DatabaseManager):
p64
=
util
.
p64
r
=
self
.
query
(
"SELECT tid, IF(compression < 128, LENGTH(value),"
" CAST(CONV(HEX(SUBSTR(value, 5, 4)), 16, 10) AS INT))"
" FROM obj FORCE INDEX(
`partition`
)"
" FROM obj FORCE INDEX(
PRIMARY
)"
" LEFT JOIN data ON (obj.data_id = data.id)"
" WHERE `partition` = %d AND oid = %d AND tid >= %d"
" ORDER BY tid DESC LIMIT %d, %d"
%
...
...
@@ -728,7 +738,7 @@ class MySQLDatabaseManager(DatabaseManager):
u64
=
util
.
u64
p64
=
util
.
p64
min_tid
=
u64
(
min_tid
)
r
=
self
.
query
(
'SELECT tid, oid FROM obj FORCE INDEX(
PRIMARY
)'
r
=
self
.
query
(
'SELECT tid, oid FROM obj FORCE INDEX(
tid
)'
' WHERE `partition` = %d AND tid <= %d'
' AND (tid = %d AND %d <= oid OR %d < tid)'
' ORDER BY tid ASC, oid ASC LIMIT %d'
%
(
...
...
@@ -793,7 +803,7 @@ class MySQLDatabaseManager(DatabaseManager):
q
=
self
.
query
self
.
_setPackTID
(
tid
)
for
count
,
oid
,
max_serial
in
q
(
"SELECT COUNT(*) - 1, oid, MAX(tid)"
" FROM obj FORCE INDEX(
`partition`
)"
" FROM obj FORCE INDEX(
PRIMARY
)"
" WHERE tid <= %d GROUP BY oid"
%
tid
):
partition
=
getPartition
(
oid
)
...
...
@@ -844,7 +854,7 @@ class MySQLDatabaseManager(DatabaseManager):
# last grouped value, instead of the greatest one.
r
=
self
.
query
(
"""SELECT tid, oid
FROM obj FORCE INDEX(
PRIMARY
)
FROM obj FORCE INDEX(
tid
)
WHERE `partition` = %(partition)s
AND tid <= %(max_tid)d
AND (tid > %(min_tid)d OR
...
...
neo/storage/database/sqlite.py
View file @
d289050e
...
...
@@ -68,7 +68,7 @@ class SQLiteDatabaseManager(DatabaseManager):
never be used for small requests.
"""
VERSION
=
1
VERSION
=
2
def
_parse
(
self
,
database
):
self
.
db
=
os
.
path
.
expanduser
(
database
)
...
...
@@ -117,11 +117,25 @@ class SQLiteDatabaseManager(DatabaseManager):
self
.
_checkNoUnfinishedTransactions
()
self
.
query
(
"DROP TABLE IF EXISTS ttrans"
)
def
_migrate2
(
self
,
schema_dict
,
index_dict
):
# BBB: As explained in _setup, no transactional DDL
# so let's do the same dance as for MySQL.
q
=
self
.
query
if
self
.
nonempty
(
'obj'
)
is
None
:
if
self
.
nonempty
(
'new_obj'
)
is
None
:
return
else
:
q
(
"DROP TABLE IF EXISTS new_obj"
)
q
(
schema_dict
.
pop
(
'obj'
)
%
'new_obj'
)
q
(
"INSERT INTO new_obj SELECT * FROM obj"
)
q
(
"DROP TABLE obj"
)
q
(
"ALTER TABLE new_obj RENAME TO obj"
)
def
_setup
(
self
,
dedup
=
False
):
#
SQLite does support transactional Data Definition Language statements
#
but unfortunately, the built-in Python binding automatically commits
#
between such statements. This anti-feature causes this method to be
#
relatively slow; u
nit tests enables the UNSAFE boolean flag.
#
BBB: SQLite has transactional DDL but before Python 3.6,
#
the binding automatically commits between such statements.
#
This anti-feature causes this method to be relatively slow.
#
U
nit tests enables the UNSAFE boolean flag.
self
.
_config
.
clear
()
q
=
self
.
query
schema_dict
=
OrderedDict
()
...
...
@@ -162,10 +176,10 @@ class SQLiteDatabaseManager(DatabaseManager):
tid INTEGER NOT NULL,
data_id INTEGER,
value_tid INTEGER,
PRIMARY KEY (partition,
tid, o
id))
PRIMARY KEY (partition,
oid, t
id))
"""
index_dict
[
'obj'
]
=
(
"CREATE INDEX %s ON %s(partition,
oid, t
id)"
,
"CREATE INDEX %s ON %s(partition,
tid, o
id)"
,
"CREATE INDEX %s ON %s(data_id)"
)
# The table "data" stores object data.
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment