Commit fd8126b5 authored by Ivan Tyagov's avatar Ivan Tyagov

test parallel execution

parent bc21eda3
...@@ -240,6 +240,32 @@ context.activate().DataStream_readChunkListAndTransform( \ ...@@ -240,6 +240,32 @@ context.activate().DataStream_readChunkListAndTransform( \
# final reduce job to a number # final reduce job to a number
sum(result_list) sum(result_list)
def test_01_02_ParallelTransformation(self):
"""
test parallel execution.
Note: determining row length is important in this case
"""
portal = self.portal
reference = getRandomString()
row = ','.join(['%s' %x for x in range(1000)])
number_string_list = [row]*20
real_data = '\n'.join(number_string_list)
portal.log( real_data)
ingestion_policy, data_supply, data_stream, data_array = self.stepSetupIngestion(reference)
data_stream.appendData(real_data)
self.tic()
data_stream.DataStream_transform(\
chunk_length = len(row), \
transform_script_id = 'DataStream_copyCSVToDataArray',
data_array_reference = reference,
parallelize = 1)
self.tic()
def test_02_Examples(self): def test_02_Examples(self):
""" """
......
...@@ -54,6 +54,9 @@ ...@@ -54,6 +54,9 @@
<string>W:176, 22: Unused variable \'data_supply\' (unused-variable)</string> <string>W:176, 22: Unused variable \'data_supply\' (unused-variable)</string>
<string>W:227, 8: Unused variable \'i\' (unused-variable)</string> <string>W:227, 8: Unused variable \'i\' (unused-variable)</string>
<string>W:176, 4: Unused variable \'ingestion_policy\' (unused-variable)</string> <string>W:176, 4: Unused variable \'ingestion_policy\' (unused-variable)</string>
<string>W:257, 4: Unused variable \'ingestion_policy\' (unused-variable)</string>
<string>W:257, 22: Unused variable \'data_supply\' (unused-variable)</string>
<string>W:257, 48: Unused variable \'data_array\' (unused-variable)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment