Extend test.

bt5/erp5_wendelin/ExtensionTemplateItem/portal_components/extension.erp5.Wendelin.py

@@ -11,14 +11,15 @@ def DataStream_copyCSVToDataArray(self, chunk_list, start, end, \
     Receive CSV data and transform it to a numpy array of floats.
   """
   chunk_text = ''.join(chunk_list)
-  
+
   # compensate possible offset mistmatch
   last_new_line_index = chunk_text.rfind('\n')
   offset_mismatch = len(chunk_text) - last_new_line_index -1
   start = start - offset_mismatch
   end = end - offset_mismatch
-  #self.log('%s %s %s' %(len(chunk_list), chunk_text.rfind('\n'), chunk_list))
   
+  #self.log(chunk_text)
+
   # remove offset line which is to be processed next call
   chunk_text = chunk_text[:len(chunk_text) - offset_mismatch - 1]
   
@@ -29,9 +30,8 @@ def DataStream_copyCSVToDataArray(self, chunk_list, start, end, \
     line_item_list = line.split(',')
     size_list.extend([x for x in line_item_list])
 
-  self.log(size_list)
-
   # save this value as a numpy array (for testing, only create ZBigArray for one variable)
+  #self.log(size_list)
   size_list = [float(x) for x in size_list]
   ndarray = np.array(size_list)
 

bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.py

@@ -35,6 +35,11 @@ import random
 def getRandomString():
   return 'test_%s' %''.join([random.choice(string.ascii_letters + string.digits) \
     for n in xrange(32)])
+    
+def chunks(l, n):
+  """Yield successive n-sized chunks from l."""
+  for i in xrange(0, len(l), n):
+    yield l[i:i+n]
 
 class Test(ERP5TypeTestCase):
   """
@@ -69,10 +74,10 @@ class Test(ERP5TypeTestCase):
     portal = self.portal
     request = portal.REQUEST
     
-    # simulate fluentd by setting proper values in REQUEST
     reference = getRandomString()
-    number_string = ','.join([str(x) for x in range(11)])
-    number_string_list = [number_string]*10000
+    number_string_list = []
+    for my_list in list(chunks(range(0, 100001), 10)):
+      number_string_list.append(','.join([str(x) for x in my_list]))
     real_data = '\n'.join(number_string_list)
 
     # create ingestion policy
@@ -109,7 +114,7 @@ class Test(ERP5TypeTestCase):
                                       data_supply_line_kw)
     self.tic()
     
-    # do real ingestion call
+    # simulate fluentd by setting proper values in REQUEST
     request.method = 'POST'
     data_chunk = msgpack.packb([0, real_data], use_bin_type=True)
     request.set('reference', reference)
@@ -131,7 +136,7 @@ class Test(ERP5TypeTestCase):
     self.tic()
     
     data_stream.DataStream_transform(\
-        chunk_length = 52001, \
+        chunk_length = 10450, \
         transform_script_id = 'DataStream_copyCSVToDataArray',
         data_array_reference = reference)
 
@@ -141,9 +146,12 @@ class Test(ERP5TypeTestCase):
     zarray = data_array.getArray()
     np.average(zarray)
     
-    # XXX: test that extracted array is same as input one
+    # test that extracted array contains same values as input CSV
     self.assertNotEqual(None, zarray)
-    #self.assertEqual(1, zarray.shape)
+    self.assertEqual(99999.0, np.amax(zarray, axis=0))
+    self.assertEqual(0.0, np.amin(zarray, axis=0))
+    # failing in array shape, not investigated why
+    self.assertEqual((99999,), zarray.shape)
     
   def test_02_Examples(self):
     """

bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.xml

@@ -47,10 +47,10 @@
             <value>
               <tuple>
                 <string>W: 37,  8: Unused variable \'n\' (unused-variable)</string>
-                <string>W: 59,  4: Unused variable \'scipy\' (unused-variable)</string>
-                <string>W: 61,  4: Unused variable \'pandas\' (unused-variable)</string>
-                <string>W: 60,  4: Unused variable \'sklearn\' (unused-variable)</string>
-                <string>W:107,  4: Unused variable \'data_supply\' (unused-variable)</string>
+                <string>W: 64,  4: Unused variable \'scipy\' (unused-variable)</string>
+                <string>W: 66,  4: Unused variable \'pandas\' (unused-variable)</string>
+                <string>W: 65,  4: Unused variable \'sklearn\' (unused-variable)</string>
+                <string>W:112,  4: Unused variable \'data_supply\' (unused-variable)</string>
               </tuple>
             </value>
         </item>