slapos/test/collect: Extend tests to increase coverage

  On the tests for ConsumptionReportBase, it contains a port from a test
   on slapos.toolbox done by Alain Takoudjou.

slapos/collect/reporter.py

@@ -310,12 +310,11 @@ class PartitionReport(ConsumptionReportBase):
     if not os.path.exists(data_file) or \
             os.stat(data_file).st_size == 0:
       with open(data_file, 'w') as fdata:
-        json.dump({
-          "date": time.time(),
-          "data": column_list 
-        }, fdata)
+        # We don't want to use json.dump here because we need to keep
+        # this precise order or be able to use append.
+        fdata.write('{"date": %s, "data": %s}' % (time.time(), json.dumps(column_list)))
 
-  def buildJSONMonitorReport(self):
+  def buildJSONMonitorReport(self, date_scope=None, min_time=None, max_time=None):
 
     for user in self.user_list.values():
       location = os.path.join(user.path, ".slapgrid")
@@ -344,7 +343,8 @@ class PartitionReport(ConsumptionReportBase):
           ["date, io rw counter, io cycles counter, disk used"])
   
       process_result, memory_result, io_result = \
-              self.getPartitionConsumptionStatusList(user.name)
+              self.getPartitionConsumptionStatusList(
+                user.name, date_scope=date_scope, min_time=min_time, max_time=max_time)
   
       resource_status_dict = {}
       if process_result and process_result['total_process']:
@@ -368,7 +368,9 @@ class PartitionReport(ConsumptionReportBase):
       with open(status_file, 'w') as fp:
         json.dump(resource_status_dict, fp)
   
-      resource_process_status_list = self.getPartitionProcessConsumptionList(user.name)
+      resource_process_status_list = self.getPartitionProcessConsumptionList(
+                user.name, date_scope=date_scope, min_time=min_time, max_time=max_time)
+
       if resource_process_status_list:
         with open(resource_file, 'w') as rf:
           json.dump(resource_process_status_list, rf)
@@ -474,6 +476,8 @@ class ConsumptionReport(ConsumptionReportBase):
      return xml_report_path
 
   def _getAverageFromList(self, data_list):
+    if not data_list:
+      return 0
     return sum(data_list)/len(data_list)
 
   @withDB

slapos/tests/data/2019-10-07.xml

+<?xml version='1.0' encoding='utf-8'?><journal><transaction type="Sale Packing List"><title>Eco Information for COMP-192938 </title><start_date>2019-10-07 00:00:00</start_date><stop_date>2019-10-07 23:59:59</stop_date><reference>2019-10-07-global</reference><currency></currency><payment_mode></payment_mode><category></category><arrow type="Destination"/><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average</title><reference>COMP-192938</reference><quantity>76.4371428571</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Used Memory</title><reference>COMP-192938</reference><quantity>14211174517.0</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/zero_emission_ratio</resource><title>Zero Emission Ratio</title><reference>COMP-192938</reference><quantity>-1</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser12</title><reference>slapuser12</reference><quantity>5.54571428571</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser13</title><reference>slapuser13</reference><quantity>1.0</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser10</title><reference>slapuser10</reference><quantity>18.1114285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser9</title><reference>slapuser9</reference><quantity>0.822857142857</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser2</title><reference>slapuser2</reference><quantity>5.79714285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser3</title><reference>slapuser3</reference><quantity>0.971428571429</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser6</title><reference>slapuser6</reference><quantity>1.39714285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser12</title><reference>slapuser12</reference><quantity>1726.23069196</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser13</title><reference>slapuser13</reference><quantity>793.577232143</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser10</title><reference>slapuser10</reference><quantity>4311.40703125</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser9</title><reference>slapuser9</reference><quantity>88.5420758929</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser2</title><reference>slapuser2</reference><quantity>1784.5375</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser3</title><reference>slapuser3</reference><quantity>370.342857143</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser6</title><reference>slapuser6</reference><quantity>886.900558036</quantity><price>0.0</price><VAT></VAT><category></category></movement></transaction></journal>
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource.status.json

+{"cpu_time": 59236.69, "cpu_percent": 3.2, "memory_rss": 1617.37, "memory_percent": 6.71, "io_rw_counter": 6085431306.0, "date": "2019-10-06 00:00:00", "total_process": 104, "disk_used": 0, "io_cycles_counter": 11844398924.0, "cpu_num_threads": 419.0}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_io.data.json

+{"date": 1570495649.5, "data": ["date, io rw counter, io cycles counter, disk used","2019-10-06 00:00:00, 6085431306.0, 11844398924.0, 0"]}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_memory.data.json

+{"date": 1570495649.5, "data": ["date, memory used percent, memory used","2019-10-06 00:00:00, 6.71, 1617.37"]}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_process.data.json

+{"date": 1570495649.5, "data": ["date, total process, CPU percent, CPU time, CPU threads","2019-10-06 00:00:00, 104, 3.2, 59236.69, 419.0"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource.status.json

+{"cpu_time": 204815.91, "cpu_percent": 3.1, "memory_rss": 3642.55, "memory_percent": 15.1, "io_rw_counter": 32087330832.0, "date": "2019-10-06 00:00:00", "total_process": 164, "disk_used": 0, "io_cycles_counter": 37490050852.0, "cpu_num_threads": 519.0}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_io.data.json

+{"date": 1570495649.5, "data": ["date, io rw counter, io cycles counter, disk used","2019-10-06 00:00:00, 32087330832.0, 37490050852.0, 0"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_memory.data.json

+{"date": 1570495649.5, "data": ["date, memory used percent, memory used","2019-10-06 00:00:00, 15.1, 3642.55"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_process.data.json

+{"date": 1570495649.5, "data": ["date, total process, CPU percent, CPU time, CPU threads","2019-10-06 00:00:00, 164, 3.1, 204815.91, 519.0"]}
\ No newline at end of file

slapos/tests/test_slapproxy.py

@@ -361,6 +361,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     Return response by server (a computer partition or an error)
     """
     rv = self._requestComputerPartition(*args, **kw)
+    time.sleep(0.01)
     requested_at = time.time()
     self.assertEqual(rv._status_code, 200)
     xml = rv.data