storage: new --prune-orphan option

neo/lib/config.py

@@ -130,6 +130,10 @@ class ConfigurationManager(object):
         # only from command line
         return self.argument_list.get('reset', False)
 
+    def getPruneOrphan(self):
+        # only from command line
+        return self.argument_list.get('prune_orphan', False)
+
     def getUUID(self):
         # only from command line
         uuid = self.argument_list.get('uuid', None)

neo/scripts/neostorage.py

@@ -30,6 +30,8 @@ parser.add_option('-d', '--database', help = 'database connections string')
 parser.add_option('-e', '--engine', help = 'database engine')
 parser.add_option('-w', '--wait', help='seconds to wait for backend to be '
     'available, before erroring-out (-1 = infinite)', type='float', default=0)
+parser.add_option('--prune-orphan', action='store_true', help='fix database'
+    ' by deleting unreferenced raw data, and exit (this can take a long time)')
 parser.add_option('--reset', action='store_true',
                   help='remove an existing database if any, and exit')
 
@@ -53,5 +55,7 @@ def main(args=None):
     # and then, load and run the application
     from neo.storage.app import Application
     app = Application(config)
-    if not config.getReset():
+    if config.getPruneOrphan():
+        print app.dm.pruneOrphan(), 'deleted record(s)'
+    elif not config.getReset():
         app.run()

neo/storage/database/importer.py

@@ -304,7 +304,7 @@ class ImporterDatabaseManager(DatabaseManager):
                     getPartitionTable changePartitionTable
                     getUnfinishedTIDDict dropUnfinishedData abortTransaction
                     storeTransaction lockTransaction unlockTransaction
-                    storeData _pruneData deferCommit
+                    storeData getOrphanList _pruneData deferCommit
                  """.split():
             setattr(self, x, getattr(self.db, x))
 

neo/storage/database/manager.py

@@ -409,6 +409,20 @@ class DatabaseManager(object):
         is always the case at tpc_vote.
         """
 
+    @abstract
+    def getOrphanList(self):
+        """Return the list of data id that is not referenced by the obj table
+
+        This is a repair method, and it's usually expensive.
+        There was a bug that did not free data of transactions that were
+        aborted before vote. This method is used to reclaim the wasted space.
+        """
+
+    def pruneOrphan(self):
+        n = self._pruneData(self.getOrphanList())
+        self.commit()
+        return n
+
     @abstract
     def _pruneData(self, data_id_list):
         """To be overridden by the backend to delete any unreferenced data
@@ -417,6 +431,8 @@ class DatabaseManager(object):
         - not in self._uncommitted_data
         - and not referenced by a fully-committed object (storage should have
           an index or a refcount of all data ids of all objects)
+
+        The returned value is the number of deleted rows from the data table.
         """
 
     @abstract

neo/storage/database/mysqldb.py

@@ -471,6 +471,11 @@ class MySQLDatabaseManager(DatabaseManager):
     _structLL = struct.Struct(">LL")
     _unpackLL = _structLL.unpack
 
+    def getOrphanList(self):
+        return [x for x, in self.query(
+            "SELECT id FROM data LEFT JOIN obj ON (id=data_id)"
+            " WHERE data_id IS NULL")]
+
     def _pruneData(self, data_id_list):
         data_id_list = set(data_id_list).difference(self._uncommitted_data)
         if data_id_list:
@@ -491,6 +496,8 @@ class MySQLDatabaseManager(DatabaseManager):
                 if bigid_list:
                     q("DELETE FROM bigdata WHERE id IN (%s)"
                       % ",".join(map(str, bigid_list)))
+                return len(id_list)
+        return 0
 
     def _bigData(self, value):
         bigdata_id, length = self._unpackLL(value)

neo/storage/database/sqlite.py

@@ -376,6 +376,11 @@ class SQLiteDatabaseManager(DatabaseManager):
                  packed, buffer(''.join(oid_list)),
                  buffer(user), buffer(desc), buffer(ext), u64(ttid)))
 
+    def getOrphanList(self):
+        return [x for x, in self.query(
+            "SELECT id FROM data LEFT JOIN obj ON (id=data_id)"
+            " WHERE data_id IS NULL")]
+
     def _pruneData(self, data_id_list):
         data_id_list = set(data_id_list).difference(self._uncommitted_data)
         if data_id_list:
@@ -385,6 +390,8 @@ class SQLiteDatabaseManager(DatabaseManager):
                 % ",".join(map(str, data_id_list))))
             q("DELETE FROM data WHERE id IN (%s)"
               % ",".join(map(str, data_id_list)))
+            return len(data_id_list)
+        return 0
 
     def storeData(self, checksum, data, compression,
             _dup=unique_constraint_message("data", "hash", "compression")):