Commit 910d1e91 authored by Julien Muchembled's avatar Julien Muchembled

storage: commit from time to time when truncating

This is all the more important for RocksDB that it wants to keep all
transaction work in RAM.

Once we had to truncate 40% of a 1TB MyRocks DB with 24 partitions,
4 being processed in parallel. Even when committing between partitions,
MariaDB used up to 200 GB. Without the commit, 1TB RAM would not have
been enough.
parent d14de83e
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import os, errno, socket, struct, sys, threading import os, errno, socket, struct, sys, threading, time
from collections import defaultdict from collections import defaultdict
from contextlib import contextmanager from contextlib import contextmanager
from copy import copy from copy import copy
...@@ -853,7 +853,12 @@ class DatabaseManager(object): ...@@ -853,7 +853,12 @@ class DatabaseManager(object):
assert tid, tid assert tid, tid
cell_list = [] cell_list = []
my_nid = self.getUUID() my_nid = self.getUUID()
commit = 0
for partition, state in self.iterAssignedCells(): for partition, state in self.iterAssignedCells():
if commit < time.time():
if commit:
self.commit()
commit = time.time() + 10
if state > tid: if state > tid:
cell_list.append((partition, my_nid, tid)) cell_list.append((partition, my_nid, tid))
self._deleteRange(partition, tid) self._deleteRange(partition, tid)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment