Skip to content

erp5
erp5
Commit 0a41a686 authored by Hardik Juneja's avatar Hardik Juneja
Browse files
Options

Joblib: Benchmark version 1

parent ef234da2
Hide whitespace changes
Inline Side-by-side
Showing with 323 additions and 2 deletions
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.benchmarkJoblib.py 0 → 100644
View file @ 0a41a686
import time
import random
import string
import transaction
def getRandomString():
return '%s' %''.join([random.choice(string.ascii_letters + string.digits) \
for n in xrange(4)])
def tend(self, path):
active_process = self.portal_activities.unrestrictedTraverse(path)
active_process.tend = time.time()
dt = active_process.tend - active_process.tstart
active_process.postResult(dt)
def example_bench_function(self, path):
active_process = self.portal_activities.unrestrictedTraverse(path)
reference = "energey_data_2" #'energey_data_%s' % getRandomString()
self.portal_activities.activate(activity="SQLQueue", after_tag="DataStream_copyCSVToDataArray", active_process=active_process).Base_joblibLinearRegression(path, reference)
# used to use stream and create/feed a data array
# data_stream = self.data_stream_module.energey_data
# data_stream_data = data_stream.getData()
# data_array = self.data_array_module.newContent(
# portal_type = 'Data Array',
# reference = reference)
# data_stream.DataStream_transform(\
# # chunk_length = 572344, \
# transform_script_id = 'DataStream_copyCSVToDataArray',
# data_array_reference = reference)
# data_array.validate()
# transaction.commit()
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.benchmarkJoblib.xml 0 → 100644
View file @ 0a41a686
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>benchmarkJoblib</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.benchmarkJoblib</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 8, 8: Unused variable \'n\' (unused-variable)</string>
<string>W: 4, 0: Unused import transaction (unused-import)</string>
</tuple>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/ExtensionTemplateItem/portal_components/extension.erp5.joblibUseCaseExamples.py
View file @ 0a41a686
...@@ -55,7 +55,7 @@ def example_simple_function(self, active_process_path): ...@@ -55,7 +55,7 @@ def example_simple_function(self, active_process_path):
# Use CMFActivity as a backend for joblob # Use CMFActivity as a backend for joblob
with parallel_backend('CMFActivity', active_process=active_process): with parallel_backend('CMFActivity', active_process=active_process):
result = Parallel(n_jobs=2, pre_dispatch='all', timeout=30, verbose=30)(delayed(sqrt)(i**2) for i in range(5)) result = Parallel(n_jobs=2, pre_dispatch='all', timeout=30, verbose=30)(delayed(sqrt)(i**2) for i in range(500))
# Set result value and an id to the active result and post it # Set result value and an id to the active result and post it
result = ActiveResult(result=result) result = ActiveResult(result=result)
...@@ -141,3 +141,38 @@ def example_grid_search_function(self, active_process_path): ...@@ -141,3 +141,38 @@ def example_grid_search_function(self, active_process_path):
clf.fit(X, y) clf.fit(X, y)
return 'ok', joblib.__version__, time.time() - tic return 'ok', joblib.__version__, time.time() - tic
#
# Example: Ordinary least squares Linear Regression
#
from sklearn.linear_model import LinearRegression, OrthogonalMatchingPursuitCV
def example_linear_regression_function(self, active_process_path , array_reference):
active_process = self.portal_activities.unrestrictedTraverse(active_process_path)
data_array = self.portal_catalog.getResultValue(
portal_type = "Data Array",
reference = array_reference)
data = data_array.getArray()
data = data[29:] # take off the coulum names
data = data.reshape(-1,29) # reshape to matrix
log("data.reshape(-1,29)", type(data))
idy_columns = [3] #,5,7,9,11,13,15,17,19,21]
Y = data[:, idy_columns].astype(np.float)
log("astype", type(Y))
idx_columns = [x for x in range(28) if x not in idy_columns]
idx_columns.pop(0)
X = data[:, idx_columns].astype(np.float)
log("astype", type(X))
lr = LinearRegression()
omp = OrthogonalMatchingPursuitCV()
with parallel_backend('CMFActivity', n_jobs=4, pre_dispatch='all', active_process=active_process):
lr.fit(X, Y)
with parallel_backend('multiprocessing', n_jobs=4):#, pre_dispatch='all', active_process=active_process):
omp.fit(X, Y)
log('omp.coef_', omp.coef_)
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib_examples/Base_BenchmarkTest.py 0 → 100644
View file @ 0a41a686
active_process = context.portal_activities.newActiveProcess()
path = active_process.getPhysicalPath()
context.Base_simpleBenchmarkTest(path)
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib_examples/Base_BenchmarkTest.xml 0 → 100644
View file @ 0a41a686
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>**kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_BenchmarkTest</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib_examples/Base_joblibLinearRegression.xml 0 → 100644
View file @ 0a41a686
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>example_linear_regression_function</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>joblibUseCaseExamples</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_joblibLinearRegression</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/SkinTemplateItem/portal_skins/erp5_joblib_examples/Base_simpleBenchmarkTest.xml 0 → 100644
View file @ 0a41a686
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>example_bench_function</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>benchmarkJoblib</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Base_simpleBenchmarkTest</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>example_bench_function</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_joblib/bt/template_extension_id_list
View file @ 0a41a686
extension.erp5.joblibUseCaseExamples extension.erp5.joblibUseCaseExamples
\ No newline at end of file extension.erp5.benchmarkJoblib
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7