Skip to content

erp5
erp5
Commit 2a3e619a authored by Hardik Juneja's avatar Hardik Juneja Committed by Hardik Juneja
Browse files
Options

[erp5_joblib]: Add tool and script

parent eb5b8287
Hide whitespace changes
Inline Side-by-side
Showing with 564 additions and 0 deletions
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.joblibFunction.py 0 → 100644
View file @ 2a3e619a
from sklearn.externals.joblib.parallel import parallel_backend, Parallel, delayed
from Products.ERP5Type.Log import log
from Products.CMFActivity.ActiveResult import ActiveResult
import time
from math import sqrt
def abc(num):
time.sleep(2)
return sqrt(num)
def test(self, active_process_path):
active_process = self.portal_activities.unrestrictedTraverse(active_process_path)
with parallel_backend('CMFActivity', active_process=active_process):
result = Parallel(n_jobs=2, pre_dispatch='all', timeout=30, verbose=30)(delayed(abc)(i**2) for i in range(20))
log("I am here", result)
return result
\ No newline at end of file
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.joblibFunction.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>joblibFunction</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.joblibFunction</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 3, 0: Unused ActiveResult imported from Products.CMFActivity.ActiveResult (unused-import)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.joblibGridSearch.py 0 → 100644
View file @ 2a3e619a
import time
import numpy as np
from Products.ERP5Type.Log import log
import sklearn
from sklearn.externals import joblib
from sklearn.externals.joblib.parallel import parallel_backend
from sklearn.datasets import load_digits
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
def test(self, active_process_path):
digits = load_digits()
X, y = digits.data, digits.target
param_grid = {
'C': np.logspace(-10, 10, 3),
'gamma': np.logspace(-10, 10, 3),
'tol': [1e-4]
}
X = np.ascontiguousarray(X)
y = np.ascontiguousarray(y)
clf = GridSearchCV(SVC(), param_grid=param_grid, verbose=10)
active_process = self.portal_activities.unrestrictedTraverse(active_process_path)
tic = time.time()
with parallel_backend('CMFActivity', n_jobs=2, active_process=active_process):
clf.fit(X, y)
log("I am here", time.time()-tic)
return 'ok', sklearn.__version__, joblib.__version__, time.time() - tic
\ No newline at end of file
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.joblibGridSearch.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>joblibGridSearch</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.joblibGridSearch</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Folder" module="OFS.Folder"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>erp5_joblib</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_callSafeFunction.py 0 → 100644
View file @ 2a3e619a
if REQUEST is not None:
raise Unauthorized
from Products.ERP5Type.Log import log
from Products.CMFActivity.ActiveResult import ActiveResult
log("Executing batch_function on %s" % context.getRelativeUrl())
result = batch_function()
log("Result of batch_function on %s: %s" % (context.getRelativeUrl(), result))
return ActiveResult(result=result, sig=hash)
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_callSafeFunction.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>hash, batch_function, REQUEST=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_callSafeFunction</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_driverScript.py 0 → 100644
View file @ 2a3e619a
import time
from Products.ERP5Type.Log import log
timeout = 10
active_process = context.portal_activities.newActiveProcess()
active_process.useBTree()
active_process_id = active_process.getId()
path = active_process.getPhysicalPath()
context.portal_activities.activate(activity="SQLQueue", after_method_id="Base_callSafeFunction", active_process=active_process, tag='abc').Base_joblibRandomForestFunction(path)
return path
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_driverScript.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>REQUEST=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_driverScript</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_joblibFunction.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>test</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>joblibFunction</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_joblibFunction</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_joblibGridSearchFunction.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>test</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>joblibGridSearch</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_joblibGridSearchFunction</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib/Base_joblibRandomForestFunction.xml 0 → 100644
View file @ 2a3e619a
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>test_function</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>joblibRandomForest</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_joblibRandomForestFunction</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/bt/description 0 → 100644
View file @ 2a3e619a
Tools for Joblib and CMF Activity Integration
\ No newline at end of file
bt5/erp5_joblib/bt/template_extension_id_list 0 → 100644
View file @ 2a3e619a
extension.erp5.joblibFunction
extension.erp5.joblibGridSearch
\ No newline at end of file
bt5/erp5_joblib/bt/template_skin_id_list 0 → 100644
View file @ 2a3e619a
erp5_joblib
\ No newline at end of file
bt5/erp5_joblib/bt/title 0 → 100644
View file @ 2a3e619a
erp5_joblib
\ No newline at end of file
bt5/erp5_joblib/bt/version 0 → 100644
View file @ 2a3e619a
0.1
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7