CMFActivity: Remove remaining transcaions.commit and move BTree to ActiveProcess

product/CMFActivity/ActiveProcess.py

@@ -32,6 +32,7 @@ from Products.CMFCore import permissions as CMFCorePermissions
 from Products.ERP5Type.Base import Base
 from Products.ERP5Type import PropertySheet
 from Products.ERP5Type.ConflictFree import ConflictFreeLog
+from BTrees.LOBTree import LOBTree
 from BTrees.Length import Length
 from random import randrange
 from .ActiveResult import ActiveResult
@@ -85,6 +86,15 @@ class ActiveProcess(Base):
   def __init__(self, *args, **kw):
     Base.__init__(self, *args, **kw)
     self.result_list = ConflictFreeLog()
+    self.use_btree = False
+
+  security.declareProtected(CMFCorePermissions.ManagePortal, 'useBTree')
+  def useBTree(self):
+    # Use BTree instead of Linked List
+    # this is used by joblib Backend to store results in a dictionary with
+    # signature as key
+    self.use_btree = True
+    self.result_list = LOBTree()
 
   security.declareProtected(CMFCorePermissions.ManagePortal, 'postResult')
   def postResult(self, result):
@@ -92,8 +102,19 @@ class ActiveProcess(Base):
       result_list = self.result_list
     except AttributeError:
       # BBB: self was created before implementation of __init__
-      self.result_list = result_list = ConflictFreeLog()
+      if self.use_btree:
+        self.result_list = result_list = LOBTree()
+      else:
+        self.result_list = result_list = ConflictFreeLog()
     else:
+      if self.use_btree:
+        if not hasattr(result, 'sig'):
+          result_id = randrange(0, 10000 * (id(result) + 1))
+        else:
+          result_id = result.sig
+        result_list.insert(result_id, result)
+        return
+
       if type(result_list) is not ConflictFreeLog: # BBB: result_list is IOBTree
         # use a random id in order to store result in a way with
         # fewer conflict errors
@@ -103,7 +124,12 @@ class ActiveProcess(Base):
         result_list[random_id] = result
         self.result_len.change(1)
         return
-    result_list.append(result)
+
+    if self.use_btree:
+      signature = int(result.sig, 16)
+      result_list.insert(signature, result)
+    else:
+      result_list.append(result)
 
   security.declareProtected(CMFCorePermissions.ManagePortal, 'postActiveResult')
   def postActiveResult(self, *args, **kw):
@@ -124,6 +150,18 @@ class ActiveProcess(Base):
       return result_list.values()
     return list(result_list)
 
+  security.declareProtected(CMFCorePermissions.ManagePortal, 'getResult')
+  def getResult(self, key, **kw):
+    """
+      Returns the result with requested key else None
+    """
+    try:
+      result_list = self.result_list
+      result = result_list[key]
+    except:
+      return None
+    return result
+
   security.declareProtected(CMFCorePermissions.ManagePortal, 'activateResult')
   def activateResult(self, result):
     if result not in (None, 0, '', (), []):

product/CMFActivity/Activity/SQLBase.py

@@ -139,7 +139,6 @@ class SQLBase(Queue):
       serialization_tag_list = [m.activity_kw.get('serialization_tag', '')
                                 for m in message_list]
       processing_node_list = []
-
       for m in message_list:
         m.order_validation_text = x = self.getOrderValidationText(m)
         processing_node_list.append(0 if x == 'none' else -1)

product/CMFActivity/ActivityJoblibBackend.py

@@ -26,16 +26,17 @@
 ##############################################################################
 ENABLE_JOBLIB = True
 
+import copy
+import hashlib
 import sys
 import time
 
-import transaction
-from BTrees.OOBTree import OOBTree
 from zLOG import LOG, INFO, WARNING
 from ZODB.POSException import ConflictError
 
 try:
   from sklearn.externals.joblib import register_parallel_backend
+  from sklearn.externals.joblib.hashing import hash as joblib_hash
   from sklearn.externals.joblib.parallel import ParallelBackendBase, parallel_backend
   from sklearn.externals.joblib.parallel import FallbackToBackend, SequentialBackend
   from sklearn.externals.joblib._parallel_backends import SafeFunction
@@ -45,13 +46,11 @@ except ImportError:
   LOG("CMFActivityBackend", WARNING, "CLASS NOT LOADED!!!")
   ENABLE_JOBLIB = False
 
-from Activity.SQLJoblib import MyBatchedSignature
-
 if ENABLE_JOBLIB:
+
   class MySafeFunction(SafeFunction):
     """Wrapper around a SafeFunction that catches any exception
-  
-    The exception can be handled in CMFActivityResult.get
+       The exception can be handled in CMFActivityResult.get
     """
     def __init__(self, *args, **kwargs):
       super(MySafeFunction, self).__init__(*args, **kwargs)
@@ -67,23 +66,12 @@ if ENABLE_JOBLIB:
       self.active_process = active_process
       self.active_process_sig = active_process_sig
       self.callback = callback
+ 
     def get(self, timeout=None):
-      
-      '''
-      while not self.active_process.getResultList():
-        time.sleep(1)
-        if timeout is not None:
-          timeout -= 1
-          if timeout < 0:
-            raise RuntimeError('Timeout reached')
-        transaction.commit()
-      '''
-
-      if self.active_process.process_result_map[self.active_process_sig] is None:
+      if self.active_process.getResult(self.active_process_sig) is None:
         raise ConflictError
-      result = self.active_process.process_result_map[self.active_process_sig]
+      result = self.active_process.getResult(self.active_process_sig).result
 
-      # TODO raise before or after the callback?
       if isinstance(result, Exception):
         raise result
       if self.callback is not None:
@@ -94,10 +82,7 @@ if ENABLE_JOBLIB:
     def __init__(self, *args, **kwargs):
       self.count = 1
       self.active_process = kwargs['active_process']
-      if not hasattr(self.active_process, 'process_result_map'):
-        self.active_process.process_result_map = OOBTree()
-      transaction.commit()
- 
+
     def effective_n_jobs(self, n_jobs):
       """Dummy implementation to prevent n_jobs <=0
 
@@ -113,14 +98,16 @@ if ENABLE_JOBLIB:
       active_process_id = self.active_process.getId()
       joblib_result = None
 
-      sig = MyBatchedSignature(batch)
-      if not self.active_process.process_result_map.has_key(sig):
-        self.active_process.process_result_map.insert(sig, None)
+      # create a signature and convert it to integer
+      sig = joblib_hash(batch.items[0])
+      sigint = int(sig, 16) % (10 ** 16)
+      if not self.active_process.getResult(sigint):
         joblib_result = portal_activities.activate(activity='SQLJoblib',
           tag="joblib_%s" % active_process_id,
-          active_process=self.active_process).Base_callSafeFunction(MySafeFunction(batch))
+          signature=sig,
+          active_process=self.active_process).Base_callSafeFunction(sigint, MySafeFunction(batch))
       if joblib_result is None:
-        joblib_result = CMFActivityResult(self.active_process, sig, callback)
+        joblib_result = CMFActivityResult(self.active_process, sigint, callback)
       return joblib_result
 
     def configure(self, n_jobs=1, parallel=None, **backend_args):
@@ -138,11 +125,6 @@ if ENABLE_JOBLIB:
 
     def abort_everything(self, ensure_ready=True):
       # All jobs will be aborted here while they are still processing our backend
-
-      # remove job with no results
-      #self.active_process.process_result_map = dict((k, v) 
-      #  for k, v in self.active_process.process_result_map.iteritems() if v)
-      transaction.commit()
       if ensure_ready:
         self.configure(n_jobs=self.parallel.n_jobs, parallel=self.parallel,
                         **self.parallel._backend_args)

product/CMFActivity/ActivityTool.py

@@ -177,7 +177,6 @@ class Message(BaseMessage):
     self.method_id = method_id
     self.args = args
     self.kw = kw
-    self.result = None
     if getattr(portal_activities, 'activity_creation_trace', False):
       # Save current traceback, to make it possible to tell where a message
       # was generated.
@@ -316,12 +315,12 @@ class Message(BaseMessage):
             result = method(*self.args, **self.kw)
         finally:
           setSecurityManager(old_security_manager)
+
         if method is not None:
           if self.active_process and result is not None:
             self.activateResult(
               activity_tool.unrestrictedTraverse(self.active_process),
               result, obj)
-          self.result = result
           self.setExecutionState(MESSAGE_EXECUTED)
     except:
       self.setExecutionState(MESSAGE_NOT_EXECUTED, context=activity_tool)
@@ -504,7 +503,6 @@ class Method(object):
       request=self._request,
       portal_activities=portal_activities,
     )
-
     if portal_activities.activity_tracking:
       activity_tracking_logger.info('queuing message: activity=%s, object_path=%s, method_id=%s, args=%s, kw=%s, activity_kw=%s, user_name=%s' % (self._activity, '/'.join(m.object_path), m.method_id, m.args, m.kw, m.activity_kw, m.user_name))
     portal_activities.getActivityBuffer().deferredQueueMessage(
@@ -1065,6 +1063,7 @@ class ActivityTool (Folder, UniqueObject):
         processing_node starts from 1 (there is not node 0)
       """
       global active_threads
+
       # return if the number of threads is too high
       # else, increase the number of active_threads and continue
       tic_lock.acquire()

product/CMFActivity/skins/activity/SQLJoblib_createMessageTable.zsql

@@ -20,6 +20,7 @@ CREATE TABLE <dtml-var table> (
   `priority` TINYINT NOT NULL DEFAULT 0,
   `group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
   `tag` VARCHAR(255) NOT NULL,
+  `signature` VARCHAR(255) NOT NULL,
   `serialization_tag` VARCHAR(255) NOT NULL,
   `retry` TINYINT UNSIGNED NOT NULL DEFAULT 0,
   `message` LONGBLOB NOT NULL,

product/CMFActivity/skins/activity/SQLJoblib_reserveDuplicatedLineList.zsql

+<dtml-comment>
+title:
+connection_id:cmf_activity_sql_connection
+max_rows:0
+max_cache:0
+cache_time:0
+class_name:
+class_file:
+</dtml-comment>
+<params>
+processing_node
+uid
+</params>
+UPDATE
+  message_job
+SET
+  processing_node=<dtml-sqlvar processing_node type="int">
+WHERE
+  <dtml-sqltest uid type="int" multiple>
+<dtml-var sql_delimiter>
+COMMIT

product/CMFActivity/skins/activity/SQLJoblib_selectDuplicatedLineList.zsql

+<dtml-comment>
+title:
+connection_id:cmf_activity_sql_connection
+max_rows:0
+max_cache:0
+cache_time:0
+class_name:
+class_file:
+</dtml-comment>
+<params>
+path
+method_id
+group_method_id
+signature
+</params>
+SELECT uid FROM
+  message_job
+WHERE
+  processing_node = 0
+  AND path = <dtml-sqlvar path type="string">
+  AND method_id = <dtml-sqlvar method_id type="string">
+  AND group_method_id = <dtml-sqlvar group_method_id type="string">
+  AND signature = <dtml-sqlvar signature type="string">
+FOR UPDATE

product/CMFActivity/skins/activity/SQLJoblib_writeMessage.zsql

@@ -18,10 +18,11 @@ processing_node
 date
 group_method_id
 tag
+signature
 serialization_tag
 </params>
 INSERT INTO <dtml-var table>
-(uid, path, active_process_uid, date, method_id, processing_node, processing, priority, group_method_id, tag, serialization_tag, message)
+(uid, path, active_process_uid, date, method_id, processing_node, processing, priority, group_method_id, tag, signature, serialization_tag, message)
 VALUES
 (
 <dtml-sqlvar expr="uid" type="int">,
@@ -34,6 +35,7 @@ VALUES
 <dtml-sqlvar expr="priority" type="int">,
 <dtml-sqlvar expr="group_method_id" type="string">,
 <dtml-sqlvar expr="tag" type="string">,
+<dtml-sqlvar expr="signature" type="string">,
 <dtml-sqlvar expr="serialization_tag" type="string">,
 <dtml-sqlvar expr="message" type="string">
 )