Commit e96e8f0b authored by Julien Muchembled's avatar Julien Muchembled

tests: fix occasional deadlock when a threaded unit test ends

During 'NEOCluster.stop', there's often 1 'Serialized.background()' call too
many, and when it removed the last lock of the queue, another thread would hang
if was sleeping between 'q.put(lock)' and 'q.get().release()'.

An easy way to reproduce the bug was to add:
    if cls.pending is None:
        time.sleep(.01)
just before 'q.get().release()'
parent bbd591f8
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
import os, random, socket, sys, tempfile, threading, time, types, weakref import os, random, socket, sys, tempfile, threading, time, types, weakref
from collections import deque from collections import deque
from functools import wraps from functools import wraps
from Queue import Queue, Empty
from mock import Mock from mock import Mock
import transaction, ZODB import transaction, ZODB
import neo.admin.app, neo.master.app, neo.storage.app import neo.admin.app, neo.master.app, neo.storage.app
...@@ -51,9 +50,8 @@ class Serialized(object): ...@@ -51,9 +50,8 @@ class Serialized(object):
def init(cls): def init(cls):
cls._global_lock = threading.Lock() cls._global_lock = threading.Lock()
cls._global_lock.acquire() cls._global_lock.acquire()
# TODO: use something else than Queue, for inspection or editing cls._lock_list = deque()
# (e.g. we'd like to suspend nodes temporarily) cls._lock_lock = threading.Lock()
cls._lock_list = Queue()
cls._pdb = False cls._pdb = False
cls.pending = 0 cls.pending = 0
...@@ -72,9 +70,14 @@ class Serialized(object): ...@@ -72,9 +70,14 @@ class Serialized(object):
except AttributeError: except AttributeError:
pass pass
q = cls._lock_list q = cls._lock_list
q.put(lock) l = cls._lock_lock
if wake_other: l.acquire()
q.get().release() try:
q.append(lock)
if wake_other:
q.popleft().release()
finally:
l.release()
@classmethod @classmethod
def acquire(cls, lock=None): def acquire(cls, lock=None):
...@@ -103,10 +106,12 @@ class Serialized(object): ...@@ -103,10 +106,12 @@ class Serialized(object):
@classmethod @classmethod
def background(cls): def background(cls):
cls._lock_lock.acquire()
try: try:
cls._lock_list.get(0).release() if cls._lock_list:
except Empty: cls._lock_list.popleft().release()
pass finally:
cls._lock_lock.release()
class SerializedEventManager(EventManager): class SerializedEventManager(EventManager):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment