Skip to content

S
slapos.core
Commit dc6e39cb authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Merge branch 'garbage-collect' into master

parents bd93529d 3ecc8bbf
Show whitespace changes
Inline Side-by-side
Showing with 1023 additions and 16 deletions
master/bt5/slapos_cloud/PathTemplateItem/portal_alarms/slapos_garbage_collect_destroy_unlinked_instance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Alarm" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>active_sense_method_id</string> </key>
<value> <string>Alarm_garbageCollectDestroyUnlinkedInstance</string> </value>
</item>
<item>
<key> <string>automatic_solve</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>slapos_garbage_collect_destroy_unlinked_instance</string> </value>
</item>
<item>
<key> <string>periodicity_hour</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_hour_frequency</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>periodicity_minute</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_minute_frequency</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>periodicity_month</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_month_day</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_start_date</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>15638400.0</float>
<string>GMT</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>periodicity_week</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Alarm</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Garbage Collect Unlinked Instances</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Alarm_garbageCollectDestroyUnlinkedInstance.py 0 → 100644
View file @ dc6e39cb
portal = context.getPortalObject()
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery
portal.portal_catalog.searchAndActivate(
portal_type=["Software Instance", "Slave Instance"],
validation_state="validated",
specialise_validation_state="validated",
predecessor_related_uid=SimpleQuery(predecessor_related_uid=None, comparison_operator='is'),
method_id='SoftwareInstance_tryToGarbageUnlinkedInstance',
activate_kw={'tag': tag}
)
context.activate(after_tag=tag).getId()
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Alarm_garbageCollectDestroyUnlinkedInstance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tag, fixit, params</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Alarm_garbageCollectDestroyUnlinkedInstance</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Instance_tryToGarbageCollect.py
View file @ dc6e39cb
......@@ -5,9 +5,6 @@ if (instance.getSlapState() != "destroy_requested"):
if (hosting_subscription.getValidationState() == "archived"):
# Buildout didn't propagate the destruction request
requester = instance.getPredecessorRelatedValue()
if (instance.getRelativeUrl() in requester.getPredecessorList()) and \
(requester.getSlapState() == "destroy_requested"):
# For security, only destroyed if parent is also destroyed
if instance.getPortalType() == 'Software Instance':
is_slave = False
......@@ -17,6 +14,22 @@ if (instance.getSlapState() != "destroy_requested"):
raise NotImplementedError, "Unknown portal type %s of %s" % \
(instance.getPortalType(), instance.getRelativeUrl())
if requester is None:
# This instance has no predecessor (link removed) and should be trashed
promise_kw = {
'instance_xml': instance.getTextContent(),
'software_type': instance.getSourceReference(),
'sla_xml': instance.getSlaXml(),
'software_release': instance.getUrlString(),
'shared': is_slave,
}
instance.requestDestroy(**promise_kw)
# Unlink all children of this instance
instance.edit(predecessor="", comment="Destroyed garbage collector!")
elif (instance.getRelativeUrl() in requester.getPredecessorList()) and \
(requester.getSlapState() == "destroy_requested"):
# For security, only destroyed if parent is also destroyed
requester.requestInstance(
software_release=instance.getUrlString(),
software_title=instance.getTitle(),
......
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Person_findPartition.py
View file @ dc6e39cb
......@@ -44,8 +44,13 @@ if computer_network_query:
else:
query_kw["default_subordination_reference"] = computer_network_query
if "retention_delay" in filter_kw:
filter_kw.pop("retention_delay")
extra_item_list = ["retention_delay",
"fw_restricted_access",
"fw_rejected_sources",
"fw_authorized_sources"]
for item in extra_item_list:
if item in filter_kw:
filter_kw.pop(item)
computer_base_category_list = [
'group',
......
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/SoftwareInstance_tryToGarbageUnlinkedInstance.py 0 → 100644
View file @ dc6e39cb
from zExceptions import Unauthorized
from DateTime import DateTime
from Products.ERP5Type.DateUtils import addToDate
if REQUEST is not None:
raise Unauthorized
instance = context
def checkInstanceTree(instance_list):
"""
Check if predecessor link is really removed to this instance
"""
sub_instance_list = []
if instance_list == []:
return
for item in instance_list:
if item.getUid() == instance.getUid():
return item
sub_instance_list.extend(item.getPredecessorValueList())
return checkInstanceTree(sub_instance_list)
if instance.getSlapState() == "destroy_requested":
return
hosting_subscription = instance.getSpecialiseValue()
if hosting_subscription is None or \
hosting_subscription.getSlapState() == "destroy_requested":
return
root_instance = hosting_subscription.getPredecessorValue()
if root_instance is None:
# Refuse to destroy root instance
raise ValueError("Hosting Subscription %s has no root instance, this should "\
"not happen!!" % hosting_subscription.getRelativeUrl())
# If instance modificationDate is too recent, skip
# Delay destroy of unlinked instances
if instance.getModificationDate() - addToDate(DateTime(), {'minute': -1*delay_time}) > 0:
return
if checkInstanceTree([root_instance]) is None:
# This unlinked instance to parent should be removed
is_slave = False
if instance.getPortalType() == 'Slave Instance':
is_slave = True
promise_kw = {
'instance_xml': instance.getTextContent(),
'software_type': instance.getSourceReference(),
'sla_xml': instance.getSlaXml(),
'software_release': instance.getUrlString(),
'shared': is_slave,
}
instance.requestDestroy(**promise_kw)
# Unlink all children of this instance
instance.edit(predecessor="", comment="Destroyed garbage collector!")
return instance.getRelativeUrl()
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/SoftwareInstance_tryToGarbageUnlinkedInstance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>REQUEST=None, delay_time=50</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SoftwareInstance_tryToGarbageUnlinkedInstance</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/TestTemplateItem/portal_components/test.erp5.testSlapOSCloudAlarm.py
View file @ dc6e39cb
......@@ -5,6 +5,7 @@ from Products.SlapOS.tests.testSlapOSMixin import \
from Products.ERP5Type.tests.utils import createZODBPythonScript
from unittest import skip
import json
import time
from zExceptions import Unauthorized
from DateTime import DateTime
from Products.ERP5Type.DateUtils import addToDate
......@@ -1389,6 +1390,62 @@ class TestSlapOSGarbageCollectDestroyedRootTreeAlarm(testSlapOSMixin):
self.assertEqual('validated',
self.requested_software_instance.getValidationState())
def test_Instance_tryToGarbageCollect_unlinked_predecessor(self):
self.requested_software_instance.edit(predecessor_list=[])
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(self.software_instance,
'destroy_requested')
self.tic()
self.requested_software_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual('destroy_requested',
self.requested_software_instance.getSlapState())
self.assertEqual('validated',
self.requested_software_instance.getValidationState())
def test_Instance_tryToGarbageCollect_destroy_unlinked_with_child(self):
instance_kw = dict(software_release=self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Sub Instance',
state='started'
)
self.requested_software_instance.requestInstance(**instance_kw)
sub_instance = self.requested_software_instance.getPredecessorValue()
self.assertNotEqual(sub_instance, None)
self.requested_software_instance.edit(predecessor_list=[])
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(self.software_instance,
'destroy_requested')
self.tic()
self.requested_software_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual('destroy_requested',
self.requested_software_instance.getSlapState())
self.assertEqual('validated',
self.requested_software_instance.getValidationState())
self.assertEqual(self.requested_software_instance.getPredecessorValue(),
None)
self.assertEqual(sub_instance.getSlapState(), 'start_requested')
sub_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual(sub_instance.getSlapState(), 'destroy_requested')
self.assertEqual(sub_instance.getValidationState(), 'validated')
def _simulateInstance_tryToGarbageCollect(self):
script_name = 'Instance_tryToGarbageCollect'
if script_name in self.portal.portal_skins.custom.objectIds():
......@@ -1961,6 +2018,252 @@ portal_workflow.doActionFor(context, action='edit_action', comment='Visited by I
'Visited by Instance_tryToGarbageCollectNonAllocatedRootTree',
instance.workflow_history['edit_workflow'][-1]['comment'])
class TestSlapOSGarbageCollectUnlinkedInstanceAlarm(testSlapOSMixin):
def createInstance(self):
hosting_subscription = self.portal.hosting_subscription_module\
.template_hosting_subscription.Base_createCloneDocument(batch_mode=1)
hosting_subscription.validate()
hosting_subscription.edit(
title=self.generateNewSoftwareTitle(),
reference="TESTHS-%s" % self.generateNewId(),
)
request_kw = dict(
software_release=\
self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title=hosting_subscription.getTitle(),
state='started'
)
hosting_subscription.requestStart(**request_kw)
hosting_subscription.requestInstance(**request_kw)
self.hosting_subscription = hosting_subscription
instance = hosting_subscription.getPredecessorValue()
return instance
def createComputerPartition(self):
computer = self.portal.computer_module\
.template_computer.Base_createCloneDocument(batch_mode=1)
computer.validate()
computer.edit(
title=self.generateNewSoftwareTitle(),
reference="TESTCOMP-%s" % self.generateNewId(),
)
partition = computer.newContent(portal_type="Computer Partition")
return partition
def doRequestInstance(self, instance, title, slave=False):
instance_kw = dict(software_release=self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=slave,
software_title=title,
state='started'
)
instance.requestInstance(**instance_kw)
self.tic()
sub_instance = instance.getPredecessorValue()
partition = self.createComputerPartition()
sub_instance.edit(aggregate_value=partition)
self.tic()
self.assertEqual(self.hosting_subscription.getRelativeUrl(),
sub_instance.getSpecialise())
return sub_instance
def _simulateSoftwareInstance_tryToGarbageUnlinkedInstance(self):
script_name = 'SoftwareInstance_tryToGarbageUnlinkedInstance'
if script_name in self.portal.portal_skins.custom.objectIds():
raise ValueError('Precondition failed: %s exists in custom' % script_name)
createZODBPythonScript(self.portal.portal_skins.custom,
script_name,
'*args, **kwargs',
'# Script body\n'
"""portal_workflow = context.portal_workflow
portal_workflow.doActionFor(context, action='edit_action', comment='Visited by SoftwareInstance_tryToGarbageUnlinkedInstance') """ )
transaction.commit()
def _dropSoftwareInstance_tryToGarbageUnlinkedInstance(self):
script_name = 'SoftwareInstance_tryToGarbageUnlinkedInstance'
if script_name in self.portal.portal_skins.custom.objectIds():
self.portal.portal_skins.custom.manage_delObjects(script_name)
transaction.commit()
def test_SoftwareInstance_tryToGarbageUnlinkedInstance(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
self.assertEqual(instance0.getPredecessorRelatedTitle(), instance.getTitle())
# Remove predecessor link
instance.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance0.getPredecessorRelatedTitle(), None)
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance(delay_time=-1)
self.tic()
self.assertEqual(instance0.getSlapState(), 'destroy_requested')
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_hosting_destroyed(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance.edit(predecessor_list=[])
self.tic()
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(instance, 'destroy_requested')
self.tic()
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
# Will not be destroyed by this script
self.assertEqual(instance0.getSlapState(), 'start_requested')
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_will_unlink_children(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance_instance0 = self.doRequestInstance(instance0, 'Subinstance0')
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(),
'instance0')
instance.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance0.getPredecessorRelatedTitle(), None)
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance(delay_time=-1)
self.tic()
self.assertEqual(instance0.getSlapState(), 'destroy_requested')
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
# Link of child removed
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(), None)
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_will_delay(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance_instance0 = self.doRequestInstance(instance0, 'Subinstance0')
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(),
'instance0')
instance.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance0.getPredecessorRelatedTitle(), None)
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
self.assertEqual(instance0.getSlapState(), 'start_requested')
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
# delay a bit
time.sleep(2)
# run with delay of 3 seconds
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance(delay_time=3/60.0)
self.tic()
self.assertEqual(instance0.getSlapState(), 'destroy_requested')
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
# Link of child removed
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(), None)
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_unlinked_root(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
self.assertEqual(self.hosting_subscription.getTitle(), instance.getTitle())
# Remove predecessor link
self.hosting_subscription.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance.getPredecessorRelatedTitle(), None)
# will not destroy
self.assertRaises(
ValueError,
instance.SoftwareInstance_tryToGarbageUnlinkedInstance,
delay_time=-10)
self.tic()
self.assertEqual(instance.getSlapState(), 'start_requested')
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_not_unlinked(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance_instance0 = self.doRequestInstance(instance0, 'Subinstance0')
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(),
'instance0')
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
# Try to remove without delete predecessor link
instance_instance0.SoftwareInstance_tryToGarbageUnlinkedInstance(delay_time=-1)
self.tic()
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
def test_alarm_search_inlinked_instance(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
self.assertEqual(instance.getPredecessorReference(),
instance0.getReference())
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertNotEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
instance0.workflow_history['edit_workflow'][-1]['comment'])
# Remove predecessor link
instance.edit(predecessor_list=[])
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
instance0.workflow_history['edit_workflow'][-1]['comment'])
def test_alarm_search_inlinked_instance_slave(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
slave_instance0 = self.doRequestInstance(instance, 'slaveInstance0', True)
self.assertEqual(instance.getPredecessorTitle(), 'slaveInstance0')
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
instance.edit(predecessor_list=[])
self.tic()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
slave_instance0.workflow_history['edit_workflow'][-1]['comment'])
class TestSlapOSInvalidateDestroyedInstance(testSlapOSMixin):
def createSoftwareInstance(self):
......
master/bt5/slapos_cloud/bt/template_path_list
View file @ dc6e39cb
......@@ -14,6 +14,7 @@ portal_alarms/slapos_allocate_instance
portal_alarms/slapos_assert_hosting_subscription_predecessor
portal_alarms/slapos_cloud_invalidate_destroyed_instance
portal_alarms/slapos_free_computer_partition
portal_alarms/slapos_garbage_collect_destroy_unlinked_instance
portal_alarms/slapos_garbage_collect_destroyed_root_tree
portal_alarms/slapos_garbage_collect_non_allocated_root_tree
portal_alarms/slapos_stop_collect_instance
......
master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapTool.py
View file @ dc6e39cb
......@@ -1681,6 +1681,123 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
if os.path.exists(self.instance_request_simulator):
os.unlink(self.instance_request_simulator)
def test_updateInstancePredecessorList(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach two software instances
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
instance_kw['software_title'] = 'Instance1'
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 2)
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update with no changes
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance0</string><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update Instance0 was not requested
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_one_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach one software instance
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 1)
self.assertSameSet(['Instance0'],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_no_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
# Try with something that doesn't exist
instance_list_xml = """
<marshal>
<list id="i2"><string>instance0</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_availableComputerPartition(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
......
master/bt5/slapos_web/PathTemplateItem/web_page_module/rjs_slapos_parameter_form_js.js
View file @ dc6e39cb
......@@ -120,6 +120,7 @@
key,
div,
label,
close_span,
input,
default_value,
default_used_list = [],
......@@ -170,6 +171,11 @@
label = document.createElement("label");
label.textContent = default_value;
label.setAttribute("class", "slapos-parameter-dict-key");
close_span = document.createElement("span");
close_span.textContent = "×";
close_span.setAttribute("class", "bt_close");
close_span.setAttribute("title", "Remove this parameter section.");
label.appendChild(close_span);
default_div.appendChild(label);
default_div = render_subform(
json_field.patternProperties['.*'],
......@@ -315,6 +321,11 @@
return element;
}
function removeSubParameter(element) {
$(element).parent().parent().remove();
return false;
}
function addSubForm(element) {
var subform_json = JSON.parse(atob(element.value)),
input_text = element.parentNode.querySelector("input[type='text']"),
......@@ -344,6 +355,7 @@
field_list = g.props.element.querySelectorAll(".slapos-parameter"),
button_list = g.props.element.querySelectorAll('button.add-sub-form'),
label_list = g.props.element.querySelectorAll('label.slapos-parameter-dict-key'),
close_list = g.props.element.querySelectorAll(".bt_close"),
i,
promise_list = [];
......@@ -374,6 +386,15 @@
));
}
for (i = 0; i < close_list.length; i = i + 1) {
promise_list.push(loopEventListener(
close_list[i],
'click',
false,
removeSubParameter.bind(g, close_list [i])
));
}
return RSVP.all(promise_list);
}
......
master/bt5/slapos_web/PathTemplateItem/web_page_module/rjs_slapos_parameter_form_js.xml
View file @ dc6e39cb
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>950.63459.41632.30105</string> </value>
<value> <string>954.11326.56915.27153</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1462374087.76</float>
<float>1474892353.46</float>
<string>UTC</string>
</tuple>
</state>
......
master/bt5/slapos_web/SkinTemplateItem/portal_skins/vifib_jauks_theme/vifib_style/vifib.css.zpt
View file @ dc6e39cb
......@@ -325,6 +325,21 @@ fieldset > .subfield > label {
text-decoration: none;
}
.bt_close, .subfield .slapos-parameter-dict-key span.bt_close{
padding: 0 6px;
display: block;
float: right;
text-overflow:clip;
white-space:nowrap;
overflow: hidden;
font-size: 1.5em;
border-radius: 2px;
}
.bt_close:hover {
background: #81afab;
color: #fff;
}
.hs-short-title{
margin-left:6px;
padding-bottom: 10px;
......
master/product/Vifib/Tool/SlapTool.py
View file @ dc6e39cb
......@@ -518,6 +518,18 @@ class SlapTool(BaseTool):
connection_xml,
slave_reference)
security.declareProtected(Permissions.AccessContentsInformation,
'updateComputerPartitionRelatedInstanceList')
def updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id,
instance_reference_xml):
"""
Update Software Instance predecessor list
"""
return self._updateComputerPartitionRelatedInstanceList(computer_id,
computer_partition_id,
instance_reference_xml)
security.declareProtected(Permissions.AccessContentsInformation,
'supplySupply')
def supplySupply(self, url, computer_id, state='available'):
......@@ -1365,6 +1377,44 @@ class SlapTool(BaseTool):
software_instance._instance_guid = instance_guid
return xml_marshaller.xml_marshaller.dumps(software_instance)
@UnrestrictedMethod
def _updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id, instance_reference_xml):
"""
Update Software Instance predecessor list to match the given list. If one
instance was not requested by this computer partition, it should be removed
in the predecessor_list of this instance.
Once the link is removed, this instance will be trashed by Garbage Collect!
instance_reference_xml contain list of title of sub-instances requested by
this instance.
"""
software_instance_document = self.\
_getSoftwareInstanceForComputerPartition(computer_id,
computer_partition_id)
cache_reference = '%s-PREDLIST' % software_instance_document.getReference()
if self._getLastData(cache_reference) != instance_reference_xml:
instance_reference_list = xml_marshaller.xml_marshaller.loads(
instance_reference_xml)
current_predecessor_list = software_instance_document.getPredecessorValueList(
portal_type=['Software Instance', 'Slave Instance'])
current_predecessor_title_list = [i.getTitle() for i in
current_predecessor_list]
# If there are items to remove
if list(set(current_predecessor_title_list).difference(instance_reference_list)) != []:
predecessor_list = [instance.getRelativeUrl() for instance in
current_predecessor_list if instance.getTitle()
in instance_reference_list]
LOG('SlapTool', INFO, '%s, %s: Updating predecessor list to %s' % (
computer_id, computer_partition_id, predecessor_list), error=False)
software_instance_document.edit(predecessor_list=predecessor_list,
comment='predecessor_list edited to unlink non commited instances')
self._storeLastData(cache_reference, instance_reference_xml)
####################################################
# Internals methods
####################################################
......
slapos/grid/slapgrid.py
View file @ dc6e39cb
......@@ -50,6 +50,7 @@ from lxml import etree
from slapos.slap.slap import NotFoundError
from slapos.slap.slap import ServerError
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
from slapos.util import mkdir_p, chownDirectory, string_to_boolean
from slapos.grid.exception import BuildoutFailedError
from slapos.grid.SlapObject import Software, Partition
......@@ -666,6 +667,25 @@ stderr_logfile_backups=1
if not promise_present:
self.logger.info("No promise.")
def _endInstallationTransaction(self, computer_partition):
partition_id = computer_partition.getId()
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(self.instance_root,
partition_id,
transaction_file_name)
if os.path.exists(transaction_file_path):
with open(transaction_file_path, 'r') as tf:
try:
computer_partition.setComputerPartitionRelatedInstanceList(
[reference for reference in tf.read().split('\n') if reference]
)
except NotFoundError, e:
# Master doesn't implement this feature ?
self.logger.warning("NotFoundError: %s. \nCannot send requested instance "\
"list to master. Please check if this feature is"\
"implemented on SlapOS Master." % str(e))
def _addFirewallRule(self, rule_command):
"""
"""
......@@ -904,6 +924,14 @@ stderr_logfile_backups=1
self.logger.debug('Check if %s requires processing...' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id)
os.environ['SLAPGRID_INSTANCE_ROOT'] = self.instance_root
# Check if transaction file of this partition exists, if the file was created,
# remove it so it will be generate with this new transaction
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % computer_partition_id
transaction_file_path = os.path.join(instance_path, transaction_file_name)
if os.path.exists(transaction_file_path):
os.unlink(transaction_file_path)
# Try to get partition timestamp (last modification date)
timestamp_path = os.path.join(
......@@ -1032,6 +1060,7 @@ stderr_logfile_backups=1
partition_ip_list)
self._checkPromises(computer_partition)
computer_partition.started()
self._endInstallationTransaction(computer_partition)
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
try:
# We want to process the partition, even if stopped, because it should
......@@ -1045,6 +1074,7 @@ stderr_logfile_backups=1
# Instance has to be stopped even if buildout/reporting is wrong.
local_partition.stop()
computer_partition.stopped()
self._endInstallationTransaction(computer_partition)
elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
local_partition.stop()
if self.firewall_conf:
......
slapos/slap/interface/slap.py
View file @ dc6e39cb
......@@ -337,6 +337,14 @@ class IComputerPartition(IBuildoutController, IRequester):
computer partition.
"""
def setComputerPartitionRelatedInstanceList(instance_reference_list):
"""
Set relation between this Instance and all his children.
instance_reference_list -- list of instances requested by this Computer
Partition.
"""
class IComputer(Interface):
"""
Computer interface specification
......
slapos/slap/slap.py
View file @ dc6e39cb
......@@ -36,6 +36,7 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease",
"Supply", "OpenOrder", "NotFoundError",
"ResourceNotReady", "ServerError", "ConnectionError"]
import os
import json
import logging
import re
......@@ -67,6 +68,7 @@ fallback_logger.addHandler(fallback_handler)
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME = '.slapos-request-transaction-%s'
class SlapDocument:
def __init__(self, connection_helper=None, hateoas_navigator=None):
......@@ -81,6 +83,7 @@ class SlapRequester(SlapDocument):
"""
Abstract class that allow to factor method for subclasses that use "request()"
"""
def _requestComputerPartition(self, request_dict):
try:
xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
......@@ -406,9 +409,38 @@ class ComputerPartition(SlapRequester):
self._partition_id = partition_id
self._request_dict = request_dict
# Just create an empty file (for nothing requested yet)
self._updateTransactionFile(partition_reference=None)
def __getinitargs__(self):
return (self._computer_id, self._partition_id, )
def _updateTransactionFile(self, partition_reference=None):
"""
Store reference to all Instances requested by this Computer Parition
"""
# Environ variable set by Slapgrid while processing this partition
instance_root = os.environ.get('SLAPGRID_INSTANCE_ROOT', '')
if not instance_root or not self._partition_id:
return
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % self._partition_id
transaction_file_path = os.path.join(instance_root, self._partition_id,
transaction_file_name)
try:
if partition_reference is None:
if os.access(os.path.join(instance_root, self._partition_id), os.W_OK):
if os.path.exists(transaction_file_path):
return
transac_file = open(transaction_file_path, 'w')
transac_file.close()
else:
with open(transaction_file_path, 'a') as transac_file:
transac_file.write('%s\n' % partition_reference)
except OSError:
return
def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None,
state=None):
......@@ -440,6 +472,7 @@ class ComputerPartition(SlapRequester):
'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state),
}
self._updateTransactionFile(partition_reference)
return self._requestComputerPartition(request_dict)
def building(self):
......@@ -635,6 +668,15 @@ class ComputerPartition(SlapRequester):
)
return xml_marshaller.loads(xml)
def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
data={
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'instance_reference_xml': xml_marshaller.dumps(instance_reference_list)
}
)
def _addIpv6Brackets(url):
# if master_url contains an ipv6 without bracket, add it
# Note that this is mostly to limit specific issues with
......
slapos/tests/slap.py
View file @ dc6e39cb
......@@ -29,6 +29,7 @@ import logging
import os
import unittest
import urlparse
import tempfile
import httmock
......@@ -53,6 +54,8 @@ class SlapMixin(unittest.TestCase):
print 'Testing against SLAP server %r' % self.server_url
self.slap = slapos.slap.slap()
self.partition_id = 'PARTITION_01'
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
def tearDown(self):
pass
......@@ -786,6 +789,84 @@ class TestComputerPartition(SlapMixin):
# request was done works correctly
self.assertEqual(requested_partition_id, requested_partition.getId())
def test_request_with_slapgrid_request_transaction(self):
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
partition_id = 'PARTITION_01'
instance_root = tempfile.mkdtemp()
partition_root = os.path.join(instance_root, partition_id)
os.mkdir(partition_root)
os.environ['SLAPGRID_INSTANCE_ROOT'] = instance_root
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(partition_root, transaction_file_name)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs
and 'computer_partition_reference' in qs):
slap_partition = slapos.slap.ComputerPartition(
qs['computer_reference'][0],
qs['computer_partition_reference'][0])
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
}
elif (url.path == '/getComputerInformation'
and 'computer_id' in qs):
slap_computer = slapos.slap.Computer(qs['computer_id'][0])
slap_computer._software_release_list = []
slap_partition = slapos.slap.ComputerPartition(
qs['computer_id'][0],
partition_id)
slap_computer._computer_partition_list = [slap_partition]
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
}
elif url.path == '/requestComputerPartition':
raise RequestWasCalled
else:
return {
'status_code': 404
}
with httmock.HTTMock(handler):
self.computer_guid = self._getTestComputerId()
self.slap = slapos.slap.slap()
self.slap.initializeConnection(self.server_url)
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content = f.read()
self.assertEqual(content, '')
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'myref')
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Not override
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Request a second instance
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'mysecondref')
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEquals(list(set(content_list)), ['myref', 'mysecondref'])
def _test_new_computer_partition_state(self, state):
"""
Helper method to automate assertions of failing states on new Computer
......
slapos/tests/slapgrid.py
View file @ dc6e39cb
......@@ -51,6 +51,7 @@ from slapos.grid.utils import md5digest
from slapos.grid.watchdog import Watchdog
from slapos.grid import SlapObject
from slapos.grid.SlapObject import WATCHDOG_MARK
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
import slapos.grid.SlapObject
import httmock
......@@ -107,6 +108,8 @@ class BasicMixin(object):
self._tempdir = tempfile.mkdtemp()
self.software_root = os.path.join(self._tempdir, 'software')
self.instance_root = os.path.join(self._tempdir, 'instance')
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
logging.basicConfig(level=logging.DEBUG)
self.setSlapgrid()
......@@ -338,6 +341,8 @@ class ComputerForTest(object):
return {'status_code': 200}
if url.path == '/softwareInstanceBang':
return {'status_code': 200}
if url.path == "/updateComputerPartitionRelatedInstanceList":
return {'status_code': 200}
if url.path == '/softwareInstanceError':
instance.error_log = '\n'.join(
[
......@@ -1647,16 +1652,21 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
# Then run usage report and see if it is still working
computer.sequence = []
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# registerComputerPartition will create one more file:
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working')
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working')
self.assertEqual(computer.sequence,
......@@ -2313,3 +2323,18 @@ class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase):
rules_list = json.loads(frules.read())
self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list)
class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase):
def test_one_partition(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
instance = computer.instance_list[0]
partition = os.path.join(self.instance_root, '0')
request_list_file = os.path.join(partition,
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name)
with open(request_list_file, 'w') as f:
f.write('some partition')
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0'])
self.assertFalse(os.path.exists(request_list_file))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7