Do test by transforming only part (tail) of appended data.

bt5/erp5_wendelin/SkinTemplateItem/portal_skins/erp5_wendelin/DataStream_transform.xml

@@ -57,8 +57,6 @@
     * transform_script_id - the script which will transform data\n
     * chunk_length - the length of a chunk\n
 """\n
-data_length = context.getSize()\n
-  \n
 start = 0\n
 end  = chunk_length\n
 context.activate().DataStream_readChunkListAndTransform( \\\n
@@ -67,8 +65,6 @@ context.activate().DataStream_readChunkListAndTransform( \\\n
   chunk_length, \\\n
   transform_script_id, \\\n
   data_array_reference)\n
-\n
-return data_length\n
 </string> </value>
         </item>
         <item>

bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.py

@@ -58,7 +58,6 @@ class Test(ERP5TypeTestCase):
     # here, you can create the categories and objects your test will depend on
     pass
   
-  
   def stepSetupIngestion(self, reference):
     """
       Generic step.
@@ -106,7 +105,7 @@ class Test(ERP5TypeTestCase):
                                             version = '001')
     data_array.validate()
     self.tic()
-                                      
+
     return ingestion_policy, data_supply, data_stream, data_array
   
   def test_0_import(self): 		 
@@ -168,32 +167,48 @@ class Test(ERP5TypeTestCase):
 
     reference = getRandomString()
     number_string_list = []
-    for my_list in list(chunks(range(0, 100001), 10)):
+    for my_list in list(chunks(range(0, 10001), 10)):
       number_string_list.append(','.join([str(x) for x in my_list]))
     real_data = '\n'.join(number_string_list)
     # make sure real_data tail is also a full line
     real_data += '\n'
 
     ingestion_policy, data_supply, data_stream, data_array = self.stepSetupIngestion(reference)
+    data_stream.appendData(real_data)
+    self.tic()
     
+    self.assertEqual(None, data_array.getArray())
+
     # override DataStream_transformTail to actually do transformation on appenData
+    start = data_stream.getSize()
     script_id = 'DataStream_transformTail'
-    script_content_list = ['**kw', """
+    script_content_list = ['', """
 # created by testWendelin.test_01_1_IngestionTail
-context.DataStream_transform(\
-        chunk_length = 10450, \
-        transform_script_id = 'DataStream_copyCSVToDataArray',
-        data_array_reference = context.getReference())"""]
+start = %s
+end  = %s
+context.activate().DataStream_readChunkListAndTransform( \
+  start, \
+  end, \
+  %s, \
+  transform_script_id = 'DataStream_copyCSVToDataArray', \
+  data_array_reference=context.getReference())""" %(start, start + 10450, 10450)]
     createZODBPythonScript(portal.portal_skins.custom, script_id, *script_content_list)
+    
+    number_string_list = []
+    for my_list in list(chunks(range(10001, 200001), 10)):
+      number_string_list.append(','.join([str(x) for x in my_list]))
+    real_data = '\n'.join(number_string_list)
+    # make sure real_data tail is also a full line
+    real_data += '\n'
       
-    # append data to Data Stream and check array.
+    # append data to Data Stream and check array which should be feed now.
     data_stream.appendData(real_data)
     self.tic()
     
     # test that extracted array contains same values as input CSV
     zarray = data_array.getArray()
-    self.assertEqual(np.average(zarray), np.average(np.arange(100001)))
-    self.assertTrue(np.array_equal(zarray, np.arange(100001)))
+    self.assertEqual(np.average(zarray), np.average(np.arange(10001, 200001)))
+    self.assertTrue(np.array_equal(zarray, np.arange(10001, 200001)))
     
     # clean up script
     portal.portal_skins.custom.manage_delObjects([script_id,])
@@ -235,14 +250,12 @@ context.DataStream_transform(\
     new_array = np.arange(1,17).reshape((4,4))
     persistent_zbig_array[:,:] = new_array
     self.assertEquals(new_array.shape, persistent_zbig_array.shape)
-
-    # (enable when new wendelin.core released as it can kill system)
     self.assertTrue(np.array_equal(new_array, persistent_zbig_array))
     
     # test set element in zbig array
     persistent_zbig_array[:2, 2] = 0
     self.assertFalse(np.array_equal(new_array, persistent_zbig_array))
 
-    # resize Zbig Array (enable when new wendelin.core released as it can kill system)
+    # resize Zbig Array
     persistent_zbig_array = np.resize(persistent_zbig_array, (100,100))
     self.assertNotEquals(pure_numpy_array.shape, persistent_zbig_array.shape)
\ No newline at end of file

bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.xml

@@ -47,12 +47,12 @@
             <value>
               <tuple>
                 <string>W: 39,  8: Unused variable \'n\' (unused-variable)</string>
-                <string>W:117,  4: Unused variable \'scipy\' (unused-variable)</string>
-                <string>W:119,  4: Unused variable \'pandas\' (unused-variable)</string>
-                <string>W:118,  4: Unused variable \'sklearn\' (unused-variable)</string>
-                <string>W:137, 22: Unused variable \'data_supply\' (unused-variable)</string>
-                <string>W:177,  4: Unused variable \'ingestion_policy\' (unused-variable)</string>
-                <string>W:177, 22: Unused variable \'data_supply\' (unused-variable)</string>
+                <string>W:116,  4: Unused variable \'scipy\' (unused-variable)</string>
+                <string>W:118,  4: Unused variable \'pandas\' (unused-variable)</string>
+                <string>W:117,  4: Unused variable \'sklearn\' (unused-variable)</string>
+                <string>W:136, 22: Unused variable \'data_supply\' (unused-variable)</string>
+                <string>W:176,  4: Unused variable \'ingestion_policy\' (unused-variable)</string>
+                <string>W:176, 22: Unused variable \'data_supply\' (unused-variable)</string>
               </tuple>
             </value>
         </item>