Commit ab2ca104 authored by Julien Muchembled's avatar Julien Muchembled

wendelin-scalability: add a scenario for ZBigArray processing

parent 03135189
......@@ -10,12 +10,15 @@
# $x 8.787 0
# 0.100036621094
# 6556
#
# The 'start_process' command is similar but by growing a ZBigArray object.
# The random data has a compression ratio of 10%.
[buildout]
extends = test-common.cfg
parts += start_ingest
parts += start_ingest start_process
[start_ingest]
[start-script-common]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
......@@ -27,20 +30,35 @@ template =
_('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password")
_('reference', help='Data Stream reference')
_('mu', type=float)
_('sigma', type=float)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
options = []
def option(name, **kw):
_(name, **kw)
options.append(name)
${:options}
args = parser.parse_args()
qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
for k in options:
v = getattr(args, k)
if v is not None:
t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
% (args.site_id, '&'.join(qs)))
c.putrequest('GET', '/%s/${:script}?%s' % (args.site_id, '&'.join(qs)))
c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders()
[start_ingest]
<= start-script-common
options =
option('id', help='Data Stream id')
option('mu', type=float)
option('sigma', type=float)
option('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
script = ERP5Site_simulateFluentdIngestion
[start_process]
<= start-script-common
options =
option('id', help='Data Array id')
script = ERP5Site_dummyZBigArrayProcessing
......@@ -5,7 +5,7 @@ from random import lognormvariate
bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma,
def simulateFluentdIngestion(self, id, mu, sigma,
chunks_per_transaction=128):
from time import time
import transaction
......@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma,
+ '/ingest')
module = self['data_stream_module']
try:
data_stream = module[reference]
data_stream = module[id]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
data_stream = module.newContent(id, 'Data Stream')
transaction.commit()
pack = struct.Struct('!d').pack
......
def dummyZBigArrayProcessing(self, id):
import numpy as np
from random import randrange, sample
import transaction
module = self['data_array_module']
try:
array = module[id]
except KeyError:
array = module.newContent(id, 'Data Array')
array.initArray(shape=(0, 64), dtype=np.int32)
transaction.commit()
note = array.getPath() + '/new_data'
array = array.getArray()
rows, cols = array.shape
y = xrange(cols)
n = 10 * (2<<20) // (cols*4)
z = np.ndarray(shape=(n, cols), dtype=array.dtype)
for row in z:
for i in sample(y, 8):
row[i] = randrange(0, 1000)
while 1:
txn = transaction.begin()
np.random.shuffle(z)
rows += n
array.resize((rows, cols))
array[-n:] = z
txn.note(note)
txn.commit()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -54,7 +54,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStreamModule_getTotalSize</string> </value>
<value> <string>Module_getTotalSize</string> </value>
</item>
</dictionary>
</pickle>
......
extension.erp5.ScalabilityFluentd
\ No newline at end of file
extension.erp5.ScalabilityFluentd
extension.erp5.ScalabilityZBigArray
\ No newline at end of file
portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_dummyZBigArrayProcessing
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
portal_skins/custom/ERP5Site_simulateFluentdIngestion
portal_skins/custom/Module_getTotalSize
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment