[slapos_cloud] Add support for "unique_by_network" SLA mode.

master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Person_findPartition.xml

@@ -53,7 +53,7 @@
             <value> <string encoding="cdata"><![CDATA[
 
 import random\n
-from Products.ZSQLCatalog.SQLCatalog import SimpleQuery\n
+from Products.ZSQLCatalog.SQLCatalog import SimpleQuery, ComplexQuery\n
 person = context\n
 \n
 computer_partition = None\n
@@ -88,6 +88,15 @@ if \'network_guid\' in filter_kw:\n
   network_guid = filter_kw.pop(\'network_guid\')\n
   query_kw["default_subordination_reference"] = SimpleQuery(default_subordination_reference=network_guid)\n
 \n
+if computer_network_query:\n
+  if query_kw.get("default_subordination_reference"):\n
+    query_kw["default_subordination_reference"] = ComplexQuery(\n
+        query_kw["default_subordination_reference"],\n
+        computer_network_query\n
+    )\n
+  else:\n
+    query_kw["default_subordination_reference"] = computer_network_query\n
+\n
 computer_base_category_list = [\n
   \'group\',\n
   \'cpu_core\',\n
@@ -161,7 +170,7 @@ return computer_partition.getRelativeUrl()\n
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>software_release_url, software_type, software_instance_portal_type, filter_kw, test_mode=False</string> </value>
+            <value> <string>software_release_url, software_type, software_instance_portal_type, filter_kw, computer_network_query=None, test_mode=False</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/SoftwareInstance_tryToAllocatePartition.xml

@@ -50,7 +50,10 @@
         </item>
         <item>
             <key> <string>_body</string> </key>
-            <value> <string>from Products.DCWorkflow.DCWorkflow import ValidationFailed\n
+            <value> <string encoding="cdata"><![CDATA[
+
+from Products.DCWorkflow.DCWorkflow import ValidationFailed\n
+from Products.ZSQLCatalog.SQLCatalog import SimpleQuery, ComplexQuery\n
 from zExceptions import Unauthorized\n
 \n
 if context.getPortalType() not in (\'Software Instance\', \'Slave Instance\'):\n
@@ -63,7 +66,7 @@ def markHistory(document, comment):\n
   if last_workflow_item != comment:\n
     portal_workflow.doActionFor(document, action=\'edit_action\', comment=comment)\n
 \n
-def assignComputerPartition(software_instance):\n
+def assignComputerPartition(software_instance, hosting_subscription):\n
   computer_partition = software_instance.getAggregateValue(\n
       portal_type="Computer Partition")\n
   if computer_partition is None:\n
@@ -75,18 +78,66 @@ def assignComputerPartition(software_instance):\n
     if not person.Person_isAllowedToAllocate():\n
       raise Unauthorized(\'Allocation disallowed\')\n
 \n
+    tag = None\n
     try:\n
       sla_dict = software_instance.getSlaXmlAsDict()\n
     except Exception:\n
       # Note: it is impossible to import module exceptions from python scripts\n
       computer_partition_relative_url = None\n
     else:\n
+\n
+      # "Each instance should be allocated to a different network." (i.e at most one instance of the tree per network)\n
+      computer_network_query = None\n
+      if sla_dict.get(\'mode\', None) == \'unique_by_network\':\n
+        # Prevent creating two instances in the same computer_network\n
+        hosting_subscription_uid = hosting_subscription.getUid()\n
+        tag = "%s_inProgress" % hosting_subscription_uid\n
+        if (context.getPortalObject().portal_activities.countMessageWithTag(tag) > 0):\n
+          # The software instance is already under creation but can not be fetched from catalog\n
+          # As it is not possible to fetch informations, just ignore\n
+          markHistory(software_instance,\n
+              \'Allocation failed: blocking activites in progress for %s\' % hosting_subscription_uid)\n
+\n
+        sla_dict.pop(\'mode\')\n
+        # XXX: does NOT scale if hosting subscription contains many SoftwareInstance\n
+        hosting_subscription = software_instance.getSpecialiseValue()\n
+        software_instance_tree_list = [sql_obj.getObject() \\\n
+            for sql_obj in context.getPortalObject().portal_catalog(\n
+                portal_type=[\'Software Instance\', \'Slave Instance\'],\n
+                default_specialise_uid=hosting_subscription.getUid(),\n
+            )\n
+        ]\n
+        computer_network_query_list = []\n
+        # Don\'t deploy in computer with no network\n
+        computer_network_query_list.append(ComplexQuery(\n
+            SimpleQuery(\n
+            default_subordination_uid=\'\'),\n
+            logical_operator=\'not\',\n
+        ))\n
+        for software_instance in software_instance_tree_list:\n
+          computer_partition = software_instance.getAggregateValue()\n
+          if not computer_partition:\n
+            continue\n
+          computer_network = computer_partition.getParentValue().getSubordinationValue()\n
+          if computer_network:\n
+            computer_network_query_list.append(ComplexQuery(\n
+                SimpleQuery(\n
+                default_subordination_uid=computer_network.getUid()),\n
+                logical_operator=\'not\',\n
+            ))\n
+      \n
+        computer_network_query = ComplexQuery(*computer_network_query_list)\n
+        hosting_subscription.serialize()\n
+\n
+      elif sla_dict.get(\'mode\'):\n
+        computer_network_query = \'-1\'\n
+\n
       computer_partition_relative_url = person.Person_restrictMethodAsShadowUser(\n
         shadow_document=person,\n
         callable_object=person.Person_findPartition,\n
         argument_list=[software_instance.getUrlString(), software_instance.getSourceReference(),\n
-        software_instance.getPortalType(), sla_dict])\n
-    return computer_partition_relative_url\n
+        software_instance.getPortalType(), sla_dict, computer_network_query])\n
+    return computer_partition_relative_url, tag\n
 \n
 software_instance = context\n
 if software_instance.getValidationState() != \'validated\' \\\n
@@ -94,14 +145,25 @@ if software_instance.getValidationState() != \'validated\' \\\n
   or software_instance.getAggregateValue(portal_type=\'Computer Partition\') is not None:\n
   return\n
 \n
+hosting_subscription = software_instance.getSpecialiseValue()\n
 try:\n
-  computer_partition_url = assignComputerPartition(software_instance)\n
+  computer_partition_url, tag = assignComputerPartition(software_instance,\n
+      hosting_subscription)\n
+\n
+  # XXX: We create a dummy activity to prevent to allocations on the same network\n
+  if tag:\n
+    hosting_subscription.activate(activity="SQLQueue", tag=tag,\n
+        after_tag="allocate_%s" % computer_partition_url).getId()\n
+  \n
 except ValueError:\n
   # It was not possible to find free Computer Partition\n
   markHistory(software_instance, \'Allocation failed: no free Computer Partition\')\n
 except Unauthorized, e:\n
   # user has bad balance\n
   markHistory(software_instance, \'Allocation failed: %s\' % e)\n
+except NotImplementedError, e:\n
+  # user has bad balance\n
+  markHistory(software_instance, \'Allocation failed: %s\' % e)\n
 else:\n
   if computer_partition_url is not None:\n
     try:\n
@@ -111,7 +173,9 @@ else:\n
       markHistory(software_instance, \'Allocation failed: consistency failed\')\n
     else:\n
       software_instance.allocatePartition(computer_partition_url=computer_partition_url)\n
-</string> </value>
+
+
+]]></string> </value>
         </item>
         <item>
             <key> <string>_params</string> </key>

master/bt5/slapos_cloud/TestTemplateItem/testSlapOSCloudAlarm.py

@@ -585,6 +585,265 @@ portal_workflow.doActionFor(context, action='edit_action', comment='Visited by S
     self.assertEqual(self.partition.getRelativeUrl(),
         self.software_instance.getAggregate(portal_type='Computer Partition'))
 
+  def test_allocation_mode_unique_by_network_one_network(self):
+    """
+    Test that when mode is "unique_by_network", we deploy new instance on
+    computer network not already used by any software instance of the
+    hosting subscription.
+    Then test that we do NOT deploy new instance on
+    computer network already used by any software instance of the
+    hosting subscription.
+    """
+    sla_xml = """<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        </instance>"""
+    self._makeTree()
+    computer1, partition1 = self._makeComputer()
+    computer2, partition2 = self._makeComputer()
+    self._installSoftware(computer1, self.software_instance.getUrlString())
+    self._installSoftware(computer2, self.software_instance.getUrlString())
+
+    new_id = self.generateNewId()
+    computer_network = self.portal.computer_network_module.newContent(
+        portal_type='Computer Network',
+        title="live_test_%s" % new_id,
+        reference="live_test_%s" % new_id)
+    computer_network.validate()
+    computer1.edit(subordination_value=computer_network)
+    computer2.edit(subordination_value=computer_network)
+
+    self.assertEqual(None, self.software_instance.getAggregateValue(
+        portal_type='Computer Partition'))
+
+    software_instance2 = self.portal.software_instance_module\
+        .template_software_instance.Base_createCloneDocument(batch_mode=1)
+    software_instance2.edit(
+      title=self.generateNewSoftwareTitle(),
+      reference="TESTSI-%s" % self.generateNewId(),
+      url_string=self.software_instance.getUrlString(),
+      source_reference=self.generateNewSoftwareType(),
+      text_content=self.generateSafeXml(),
+      sla_xml=sla_xml,
+      specialise=self.hosting_subscription.getRelativeUrl(),
+    )
+    self.portal.portal_workflow._jumpToStateFor(software_instance2, 'start_requested')
+    software_instance2.validate()
+    self.tic()
+
+    self.software_instance.setSlaXml(sla_xml)
+    self.software_instance.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        computer_network.getReference(),
+        self.software_instance.getAggregateValue(portal_type='Computer Partition')\
+            .getParentValue().getSubordinationReference(),
+    )
+
+    self.tic()
+    software_instance2.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        None,
+        software_instance2.getAggregate(portal_type='Computer Partition')
+    )
+
+  def test_allocation_mode_unique_by_network_several_network(self):
+    """
+    Test that when mode is "unique_by_network", we deploy new instance on
+    computer network not already used by any software instance of the
+    hosting subscription.
+    Then test that we do NOT deploy new instance on
+    computer network already used by any software instance of the
+    hosting subscription.
+    Test with 3 instances and 3 existing computers on 2 different networks.
+    """
+    sla_xml = """<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        </instance>"""
+    self._makeTree()
+    computer1, partition1 = self._makeComputer()
+    computer2, partition2 = self._makeComputer()
+    computer3, partition3 = self._makeComputer()
+    computer_network1 = self._makeComputerNetwork()
+    computer_network2 = self._makeComputerNetwork()
+
+    computer1.edit(subordination_value=computer_network1)
+    computer2.edit(subordination_value=computer_network1)
+    computer3.edit(subordination_value=computer_network2)
+
+    self._installSoftware(computer1, self.software_instance.getUrlString())
+    self._installSoftware(computer2, self.software_instance.getUrlString())
+    self._installSoftware(computer3, self.software_instance.getUrlString())
+
+    self.assertEqual(None, self.software_instance.getAggregateValue(
+        portal_type='Computer Partition'))
+
+    self.software_instance.setSlaXml("""<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        <parameter id='computer_guid'>%s</parameter>
+        </instance>""" % computer1.getReference())
+    self.software_instance.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        self.software_instance.getAggregate(portal_type='Computer Partition'),
+        partition1.getRelativeUrl(),
+    )
+
+    software_instance2 = self.portal.software_instance_module\
+        .template_software_instance.Base_createCloneDocument(batch_mode=1)
+    software_instance2.edit(
+      title=self.generateNewSoftwareTitle(),
+      reference="TESTSI-%s" % self.generateNewId(),
+      url_string=self.software_instance.getUrlString(),
+      source_reference=self.generateNewSoftwareType(),
+      text_content=self.generateSafeXml(),
+      sla_xml=sla_xml,
+      specialise=self.hosting_subscription.getRelativeUrl(),
+    )
+    self.portal.portal_workflow._jumpToStateFor(software_instance2, 'start_requested')
+    software_instance2.validate()
+    self.tic()
+    software_instance2.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        software_instance2.getAggregate(portal_type='Computer Partition'),
+        partition3.getRelativeUrl(),
+    )
+
+    software_instance3 = self.portal.software_instance_module\
+        .template_software_instance.Base_createCloneDocument(batch_mode=1)
+    software_instance3.edit(
+      title=self.generateNewSoftwareTitle(),
+      reference="TESTSI-%s" % self.generateNewId(),
+      url_string=self.software_instance.getUrlString(),
+      source_reference=self.generateNewSoftwareType(),
+      text_content=self.generateSafeXml(),
+      sla_xml=sla_xml,
+      specialise=self.hosting_subscription.getRelativeUrl(),
+    )
+    self.portal.portal_workflow._jumpToStateFor(software_instance3, 'start_requested')
+    software_instance3.validate()
+    self.tic()
+
+    software_instance3.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        None,
+        software_instance3.getAggregate(portal_type='Computer Partition')
+    )
+
+  def test_allocation_mode_unique_by_network_no_network(self):
+    """
+    Test that when we request instance with mode as 'unique_by_network',
+    instance is not deployed on computer with no network.
+    """
+    self._makeTree()
+    self._makeComputer()
+    self._installSoftware(self.computer,
+        self.software_instance.getUrlString())
+
+    self.assertEqual(None, self.software_instance.getAggregateValue(
+        portal_type='Computer Partition'))
+
+    self.software_instance.setSlaXml("""<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        </instance>""")
+    self.software_instance.SoftwareInstance_tryToAllocatePartition()
+    self.assertEqual(
+        None,
+        self.software_instance.getAggregate(portal_type='Computer Partition')
+    )
+
+  def test_allocation_mode_unique_by_network_check_serialize_called(self):
+    """
+    Test that on being_requested serialise is being called
+    code stolen from testERP5Security:test_MultiplePersonReferenceConcurrentTransaction
+    """
+    class DummyTestException(Exception):
+      pass
+
+    def verify_serialize_call(self):
+      # it is checking that anything below computer_module raises exception
+      # thanks to this this test do not have to be destructive
+      if self.getPortalType() == "Hosting Subscription":
+        raise DummyTestException
+      else:
+        return self.serialize_call()
+
+    self._makeTree()
+    self.software_instance.setSlaXml("""<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        </instance>""")
+
+    from Products.ERP5Type.Base import Base
+    Base.serialize_call = Base.serialize
+    try:
+      Base.serialize = verify_serialize_call
+      self.assertRaises(DummyTestException,
+        self.software_instance.SoftwareInstance_tryToAllocatePartition)
+    finally:
+      Base.serialize = Base.serialize_call
+
+    transaction.abort()
+
+  def test_allocation_mode_unique_by_network_no_parallel(self):
+    """
+    Test that when we request two instances of the same Hosting Subscription
+    with mode as 'unique_by_network' at the same time, they don't get
+    allocated to the same network.
+    """
+    sla_xml = """<?xml version='1.0' encoding='utf-8'?>
+        <instance>
+        <parameter id='mode'>unique_by_network</parameter>
+        </instance>"""
+    self._makeTree()
+    computer1, partition1 = self._makeComputer()
+    computer2, partition2 = self._makeComputer()
+    self._installSoftware(computer1, self.software_instance.getUrlString())
+    self._installSoftware(computer2, self.software_instance.getUrlString())
+
+    new_id = self.generateNewId()
+    computer_network = self.portal.computer_network_module.newContent(
+        portal_type='Computer Network',
+        title="live_test_%s" % new_id,
+        reference="live_test_%s" % new_id)
+    computer_network.validate()
+    computer1.edit(subordination_value=computer_network)
+    computer2.edit(subordination_value=computer_network)
+
+    self.assertEqual(None, self.software_instance.getAggregateValue(
+        portal_type='Computer Partition'))
+
+    software_instance2 = self.portal.software_instance_module\
+        .template_software_instance.Base_createCloneDocument(batch_mode=1)
+    software_instance2.edit(
+      title=self.generateNewSoftwareTitle(),
+      reference="TESTSI-%s" % self.generateNewId(),
+      url_string=self.software_instance.getUrlString(),
+      source_reference=self.generateNewSoftwareType(),
+      text_content=self.generateSafeXml(),
+      sla_xml=sla_xml,
+      specialise=self.hosting_subscription.getRelativeUrl(),
+    )
+    self.portal.portal_workflow._jumpToStateFor(software_instance2, 'start_requested')
+    software_instance2.validate()
+    self.tic()
+
+    self.software_instance.setSlaXml(sla_xml)
+    self.software_instance.SoftwareInstance_tryToAllocatePartition()
+    software_instance2.SoftwareInstance_tryToAllocatePartition()
+    # First is deployed
+    self.assertEqual(
+        computer_network.getReference(),
+        self.software_instance.getAggregateValue(portal_type='Computer Partition')\
+            .getParentValue().getSubordinationReference(),
+    )
+    # But second is not yet deployed because of pending activities containing tag
+    self.assertEqual(
+        None,
+        software_instance2.getAggregate(portal_type='Computer Partition')
+    )
+
   def test_allocation_unexpected_sla_parameter(self):
     self._makeTree()
 
@@ -1731,4 +1990,3 @@ portal_workflow.doActionFor(context, action='edit_action', comment='Visited by S
     self.assertEqual(
         'Visited by SoftwareInstance_tryToInvalidateIfDestroyed',
         instance.workflow_history['edit_workflow'][-1]['comment'])
-