wip slapgrid tests are passing with new API

slapos/grid/slapgrid.py

@@ -574,7 +574,7 @@ stderr_logfile_backups=1
             "portal_type": "Software Instance",
             "compute_partition_id": partition.getId(),
             "state": partition.getState(),
-            "software_type": self.getInstanceParameterDict().get(
+            "software_type": partition.getInstanceParameterDict().get(
               'slap_software_type', None),
             "parameters": partition.getInstanceParameterDict(),
             "instance_processing_timestamp": partition.getInstanceParameterDict().get(

slapos/manager/portredir.py

@@ -82,9 +82,8 @@ class Manager(object):
 
     # Get partitions IPv6 address
     computer_partition = partition.computer_partition
-    parameter_dict = computer_partition.getInstanceParameterDict()
 
-    partition_ip_list = parameter_dict['ip_list'] + parameter_dict.get(
+    partition_ip_list = computer_partition['ip_list'] + computer_partition.get(
       'full_ip_list', [])
     partition_ip_list = [tup[1] for tup in partition_ip_list]
 

slapos/slap/slap.py

@@ -719,6 +719,36 @@ class SlapConnectionHelper(ConnectionHelper):
 
     return loads(xml)
 
+# https://stackoverflow.com/a/33571117
+def _byteify(data, ignore_dicts = False):
+  if isinstance(data, str):
+    return data
+
+  # if this is a list of values, return list of byteified values
+  if isinstance(data, list):
+    return [ _byteify(item, ignore_dicts=True) for item in data ]
+  # if this is a dictionary, return dictionary of byteified keys and values
+  # but only if we haven't already byteified it
+  if isinstance(data, dict) and not ignore_dicts:
+    return {
+      _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
+      for key, value in data.items() # changed to .items() for python 2.7/3
+    }
+
+  # python 3 compatible duck-typing
+  # if this is a unicode string, return its string representation
+  if str(type(data)) == "<type 'unicode'>":
+    return data.encode('utf-8')
+
+  # if it's anything else, return it in its original form
+  return data
+
+def json_loads_byteified(json_text):
+  return _byteify(
+    json.loads(json_text, object_hook=_byteify),
+    ignore_dicts=True
+  )
+
 class JioAPIConnectionHelper(ConnectionHelper):
 
   def apiCall(self, path, data):
@@ -727,7 +757,7 @@ class JioAPIConnectionHelper(ConnectionHelper):
                           data=json.dumps(data),
                           headers={'Content-type': 'application/json'},
                           expect_json_error=True)
-    return req.json()
+    return json_loads_byteified(req.text)
 
   def get(self, data):
     return self.apiCall(path="get/",

slapos/tests/test_slapgrid.py

@@ -884,6 +884,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'stopped')
       self.assertEqual(instance.state, 'stopped')
 
   def test_one_partition_instance_cfg(self):
@@ -902,6 +903,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'stopped')
       self.assertEqual(instance.state, 'stopped')
 
   def test_one_free_partition(self):
@@ -936,6 +938,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(partition.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(partition.state, 'started')
 
   def test_one_partition_started_fail(self):
@@ -954,6 +957,7 @@ class TestSlapgridCPWithMaster(MasterMixin, unittest.TestCase):
       six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(partition.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(partition.state, 'started')
 
       instance = computer.instance_list[0]
@@ -969,6 +973,7 @@ exit 1
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/', '/getHateoasUrl',
                        '/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(instance.sequence[3][1]["reported_state"], 'error')
       self.assertEqual(instance.state, 'started')
       self.assertTrue(instance.error_log.startswith("Failed to run buildout profile in direct"))
 
@@ -1008,6 +1013,7 @@ chmod 755 etc/run/wrapper
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       self.assertEqual(computer.sequence,
                        ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(instance.state, 'started')
 
       computer.sequence = []
@@ -1021,6 +1027,7 @@ chmod 755 etc/run/wrapper
       self.assertEqual(computer.sequence,
                        ['/getHateoasUrl',
                         '/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(instance.sequence[3][1]["reported_state"], 'stopped')
       self.assertEqual(instance.state, 'stopped')
 
   def test_one_broken_partition_stopped(self):
@@ -1067,6 +1074,7 @@ chmod 755 etc/run/wrapper
                        ['/api/allDocs/',
                         '/api/get/',
                         '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(instance.state, 'started')
 
       computer.sequence = []
@@ -1086,6 +1094,7 @@ exit 1
                         '/api/allDocs/',
                         '/api/get/',
                         '/api/put/'])
+      self.assertEqual(instance.sequence[3][1]["reported_state"], 'error')
       self.assertEqual(instance.state, 'started')
       self.assertTrue(instance.error_log.startswith("Failed to run buildout profile in direct"))
 
@@ -1107,6 +1116,7 @@ exit 1
                        ['/api/allDocs/',
                         '/api/get/',
                         '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'stopped')
       self.assertEqual('stopped', instance.state)
 
       instance.requested_state = 'started'
@@ -1126,6 +1136,7 @@ exit 1
                         '/api/allDocs/',
                         '/api/get/',
                         '/api/put/'])
+      self.assertEqual(instance.sequence[3][1]["reported_state"], 'started')
       self.assertEqual('started', instance.state)
 
   def test_one_partition_destroyed(self):
@@ -1152,6 +1163,7 @@ exit 1
                        ['/api/allDocs/',
                         '/api/get/',
                         '/api/put/'])
+      self.assertEqual(instance.sequence[1][1]["reported_state"], 'stopped')
       self.assertEqual('stopped', instance.state)
 
 
@@ -3077,27 +3089,30 @@ exit 0
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       self.assertEqual(computer.sequence,
-                       ['/getFullComputerInformation', 
-                        '/startedComputerPartition'])
+                       ['/api/allDocs/',
+                        '/api/get/',
+                        '/api/put/'])
+      self.assertEqual(partition.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(partition.state, 'started')
       manager_list = slapmanager.from_config({'manager_list': 'prerm'})
       self.grid._manager_list = manager_list
 
       partition.requested_state = 'destroyed'
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
       self.assertInstanceDirectoryListEqual(['0'])
       six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
                              'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
-                             '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
-                             '.slapos-request-transaction-0'])
+                             '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
       six.assertCountEqual(self, os.listdir(self.software_root),
                             [partition.software.software_hash])
 
       # wait until the pre-delete script is finished
       self._wait_prerm_script_finished(partition.partition_path)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
@@ -3128,12 +3143,13 @@ exit 0
       self.grid._manager_list = manager_list
 
       partition.requested_state = 'destroyed'
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (retention-delay-lock)
       six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay',
-                             '.slapos-retention-lock-date', '.slapos-request-transaction-0'])
+                             '.slapos-retention-lock-date'])
       self.assertTrue(os.path.exists(pre_delete_script))
       self.assertTrue(os.path.exists(os.path.join(
           partition.partition_path,
@@ -3141,6 +3157,7 @@ exit 0
       )))
 
       time.sleep(1)
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
 
       # Assert partition directory is not destroyed (pre-delete is running)
@@ -3148,7 +3165,7 @@ exit 0
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
                              'worked', '.slapos-retention-lock-delay', '.slapos-retention-lock-date',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
-                             '.slapos-request-transaction-0'])
+                            ])
 
       # wait until the pre-delete script is finished
       self._wait_prerm_script_finished(partition.partition_path)
@@ -3174,11 +3191,12 @@ exit 0
       self.grid._manager_list = manager_list
 
       partition.requested_state = 'destroyed'
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
       six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
-                             'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
+                             'worked', '.slapos-retention-lock-delay',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
 
       # wait until the pre-delete script is finished
@@ -3187,6 +3205,7 @@ exit 0
         # the script is well finished...
         self.assertTrue("finished prerm script." in f.read())
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
@@ -3209,11 +3228,12 @@ exit 0
       self.grid._manager_list = manager_list
 
       partition.requested_state = 'destroyed'
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is not destroyed (pre-delete is running)
       six.assertCountEqual(self, os.listdir(partition.partition_path),
                             ['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
-                             'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
+                             'worked', '.slapos-retention-lock-delay',
                              '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
 
       stat_info = os.stat(partition.partition_path)
@@ -3239,6 +3259,7 @@ exit 0
       # wait until the pre-delete script is finished
       self._wait_prerm_script_finished(partition.partition_path)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
       # Assert partition directory is empty
       self.assertInstanceDirectoryListEqual(['0'])
@@ -3391,8 +3412,8 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
     self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
     self.assertEqual(self.computer.sequence,
-                     ['/getFullComputerInformation', 
-                      '/startedComputerPartition'])
+                      ['/api/allDocs/', '/api/get/', '/api/put/'])
+    self.assertEqual(self.partition.sequence[1][1]["reported_state"], 'started')
     self.assertEqual(self.partition.state, 'started')
 
   def test_simple_port_redirection(self):
@@ -3466,8 +3487,13 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(self.computer.sequence,
-                       ['/getFullComputerInformation', 
-                        '/startedComputerPartition', '/startedComputerPartition'])
+                       ['/api/allDocs/',
+                        '/api/get/',
+                        '/api/put/',
+                        '/api/get/',
+                        '/api/put/'])
+      self.assertEqual(self.partition.sequence[1][1]["reported_state"], 'started')
+      self.assertEqual(self.partition.sequence[3][1]["reported_state"], 'started')
       self.assertEqual(self.partition.state, 'started')
 
       # Check the socat command
@@ -3871,6 +3897,7 @@ class TestSlapgridWithWhitelistfirewall(MasterMixin, unittest.TestCase):
       self.partition.requested_state = 'started'
       self.partition.software.setBuildout(WRAPPER_CONTENT)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(
@@ -3889,6 +3916,7 @@ class TestSlapgridWithWhitelistfirewall(MasterMixin, unittest.TestCase):
       self.partition.requested_state = 'started'
       self.partition.software.setBuildout(WRAPPER_CONTENT)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(
@@ -3913,6 +3941,7 @@ class TestSlapgridWithWhitelistfirewall(MasterMixin, unittest.TestCase):
       self.partition.requested_state = 'started'
       self.partition.software.setBuildout(WRAPPER_CONTENT)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(
@@ -3997,6 +4026,7 @@ class TestSlapgridWithWhitelistfirewall(MasterMixin, unittest.TestCase):
       self.partition.requested_state = 'destroyed'
       self.partition.software.setBuildout(WRAPPER_CONTENT)
 
+      self.grid.computer_partition_list = None
       self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(
@@ -4053,8 +4083,8 @@ class TestSlapgridManagerLifecycle(MasterMixin, unittest.TestCase):
       self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
 
       self.assertEqual(self.computer.sequence,
-                       ['/getFullComputerInformation', 
-                        '/startedComputerPartition'])
+                       ['/api/allDocs/', '/api/get/', '/api/put/'])
+      self.assertEqual(partition.sequence[1][1]["reported_state"], 'started')
       self.assertEqual(partition.state, 'started')
 
       self.assertEqual(self.manager.sequence,