Skip to content

S
slapos.core
Commit dc6e39cb authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Merge branch 'garbage-collect' into master

parents bd93529d 3ecc8bbf
Expand all Show whitespace changes
Inline Side-by-side
Showing with 1023 additions and 16 deletions
master/bt5/slapos_cloud/PathTemplateItem/portal_alarms/slapos_garbage_collect_destroy_unlinked_instance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Alarm" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>active_sense_method_id</string> </key>
<value> <string>Alarm_garbageCollectDestroyUnlinkedInstance</string> </value>
</item>
<item>
<key> <string>automatic_solve</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>slapos_garbage_collect_destroy_unlinked_instance</string> </value>
</item>
<item>
<key> <string>periodicity_hour</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_hour_frequency</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>periodicity_minute</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_minute_frequency</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>periodicity_month</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_month_day</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>periodicity_start_date</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>15638400.0</float>
<string>GMT</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>periodicity_week</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Alarm</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Garbage Collect Unlinked Instances</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Alarm_garbageCollectDestroyUnlinkedInstance.py 0 → 100644
View file @ dc6e39cb
portal = context.getPortalObject()
from Products.ZSQLCatalog.SQLCatalog import SimpleQuery
portal.portal_catalog.searchAndActivate(
portal_type=["Software Instance", "Slave Instance"],
validation_state="validated",
specialise_validation_state="validated",
predecessor_related_uid=SimpleQuery(predecessor_related_uid=None, comparison_operator='is'),
method_id='SoftwareInstance_tryToGarbageUnlinkedInstance',
activate_kw={'tag': tag}
)
context.activate(after_tag=tag).getId()
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Alarm_garbageCollectDestroyUnlinkedInstance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tag, fixit, params</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Alarm_garbageCollectDestroyUnlinkedInstance</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Instance_tryToGarbageCollect.py
View file @ dc6e39cb
...@@ -5,9 +5,6 @@ if (instance.getSlapState() != "destroy_requested"): ...@@ -5,9 +5,6 @@ if (instance.getSlapState() != "destroy_requested"):
if (hosting_subscription.getValidationState() == "archived"): if (hosting_subscription.getValidationState() == "archived"):
# Buildout didn't propagate the destruction request # Buildout didn't propagate the destruction request
requester = instance.getPredecessorRelatedValue() requester = instance.getPredecessorRelatedValue()
if (instance.getRelativeUrl() in requester.getPredecessorList()) and \
(requester.getSlapState() == "destroy_requested"):
# For security, only destroyed if parent is also destroyed
if instance.getPortalType() == 'Software Instance': if instance.getPortalType() == 'Software Instance':
is_slave = False is_slave = False
...@@ -17,6 +14,22 @@ if (instance.getSlapState() != "destroy_requested"): ...@@ -17,6 +14,22 @@ if (instance.getSlapState() != "destroy_requested"):
raise NotImplementedError, "Unknown portal type %s of %s" % \ raise NotImplementedError, "Unknown portal type %s of %s" % \
(instance.getPortalType(), instance.getRelativeUrl()) (instance.getPortalType(), instance.getRelativeUrl())
if requester is None:
# This instance has no predecessor (link removed) and should be trashed
promise_kw = {
'instance_xml': instance.getTextContent(),
'software_type': instance.getSourceReference(),
'sla_xml': instance.getSlaXml(),
'software_release': instance.getUrlString(),
'shared': is_slave,
}
instance.requestDestroy(**promise_kw)
# Unlink all children of this instance
instance.edit(predecessor="", comment="Destroyed garbage collector!")
elif (instance.getRelativeUrl() in requester.getPredecessorList()) and \
(requester.getSlapState() == "destroy_requested"):
# For security, only destroyed if parent is also destroyed
requester.requestInstance( requester.requestInstance(
software_release=instance.getUrlString(), software_release=instance.getUrlString(),
software_title=instance.getTitle(), software_title=instance.getTitle(),
......
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/Person_findPartition.py
View file @ dc6e39cb
...@@ -44,8 +44,13 @@ if computer_network_query: ...@@ -44,8 +44,13 @@ if computer_network_query:
else: else:
query_kw["default_subordination_reference"] = computer_network_query query_kw["default_subordination_reference"] = computer_network_query
if "retention_delay" in filter_kw: extra_item_list = ["retention_delay",
filter_kw.pop("retention_delay") "fw_restricted_access",
"fw_rejected_sources",
"fw_authorized_sources"]
for item in extra_item_list:
if item in filter_kw:
filter_kw.pop(item)
computer_base_category_list = [ computer_base_category_list = [
'group', 'group',
......
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/SoftwareInstance_tryToGarbageUnlinkedInstance.py 0 → 100644
View file @ dc6e39cb
from zExceptions import Unauthorized
from DateTime import DateTime
from Products.ERP5Type.DateUtils import addToDate
if REQUEST is not None:
raise Unauthorized
instance = context
def checkInstanceTree(instance_list):
"""
Check if predecessor link is really removed to this instance
"""
sub_instance_list = []
if instance_list == []:
return
for item in instance_list:
if item.getUid() == instance.getUid():
return item
sub_instance_list.extend(item.getPredecessorValueList())
return checkInstanceTree(sub_instance_list)
if instance.getSlapState() == "destroy_requested":
return
hosting_subscription = instance.getSpecialiseValue()
if hosting_subscription is None or \
hosting_subscription.getSlapState() == "destroy_requested":
return
root_instance = hosting_subscription.getPredecessorValue()
if root_instance is None:
# Refuse to destroy root instance
raise ValueError("Hosting Subscription %s has no root instance, this should "\
"not happen!!" % hosting_subscription.getRelativeUrl())
# If instance modificationDate is too recent, skip
# Delay destroy of unlinked instances
if instance.getModificationDate() - addToDate(DateTime(), {'minute': -1*delay_time}) > 0:
return
if checkInstanceTree([root_instance]) is None:
# This unlinked instance to parent should be removed
is_slave = False
if instance.getPortalType() == 'Slave Instance':
is_slave = True
promise_kw = {
'instance_xml': instance.getTextContent(),
'software_type': instance.getSourceReference(),
'sla_xml': instance.getSlaXml(),
'software_release': instance.getUrlString(),
'shared': is_slave,
}
instance.requestDestroy(**promise_kw)
# Unlink all children of this instance
instance.edit(predecessor="", comment="Destroyed garbage collector!")
return instance.getRelativeUrl()
master/bt5/slapos_cloud/SkinTemplateItem/portal_skins/slapos_cloud/SoftwareInstance_tryToGarbageUnlinkedInstance.xml 0 → 100644
View file @ dc6e39cb
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>REQUEST=None, delay_time=50</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SoftwareInstance_tryToGarbageUnlinkedInstance</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_cloud/bt/template_path_list
View file @ dc6e39cb
...@@ -14,6 +14,7 @@ portal_alarms/slapos_allocate_instance ...@@ -14,6 +14,7 @@ portal_alarms/slapos_allocate_instance
portal_alarms/slapos_assert_hosting_subscription_predecessor portal_alarms/slapos_assert_hosting_subscription_predecessor
portal_alarms/slapos_cloud_invalidate_destroyed_instance portal_alarms/slapos_cloud_invalidate_destroyed_instance
portal_alarms/slapos_free_computer_partition portal_alarms/slapos_free_computer_partition
portal_alarms/slapos_garbage_collect_destroy_unlinked_instance
portal_alarms/slapos_garbage_collect_destroyed_root_tree portal_alarms/slapos_garbage_collect_destroyed_root_tree
portal_alarms/slapos_garbage_collect_non_allocated_root_tree portal_alarms/slapos_garbage_collect_non_allocated_root_tree
portal_alarms/slapos_stop_collect_instance portal_alarms/slapos_stop_collect_instance
......
master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapTool.py
View file @ dc6e39cb
...@@ -1681,6 +1681,123 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin): ...@@ -1681,6 +1681,123 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
if os.path.exists(self.instance_request_simulator): if os.path.exists(self.instance_request_simulator):
os.unlink(self.instance_request_simulator) os.unlink(self.instance_request_simulator)
def test_updateInstancePredecessorList(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach two software instances
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
instance_kw['software_title'] = 'Instance1'
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 2)
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update with no changes
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance0</string><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update Instance0 was not requested
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_one_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach one software instance
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 1)
self.assertSameSet(['Instance0'],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_no_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
# Try with something that doesn't exist
instance_list_xml = """
<marshal>
<list id="i2"><string>instance0</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_availableComputerPartition(self): def test_availableComputerPartition(self):
self._makeComplexComputer() self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue( partition_id = self.start_requested_software_instance.getAggregateValue(
......
master/bt5/slapos_web/PathTemplateItem/web_page_module/rjs_slapos_parameter_form_js.js
View file @ dc6e39cb
...@@ -120,6 +120,7 @@ ...@@ -120,6 +120,7 @@
key, key,
div, div,
label, label,
close_span,
input, input,
default_value, default_value,
default_used_list = [], default_used_list = [],
...@@ -170,6 +171,11 @@ ...@@ -170,6 +171,11 @@
label = document.createElement("label"); label = document.createElement("label");
label.textContent = default_value; label.textContent = default_value;
label.setAttribute("class", "slapos-parameter-dict-key"); label.setAttribute("class", "slapos-parameter-dict-key");
close_span = document.createElement("span");
close_span.textContent = "×";
close_span.setAttribute("class", "bt_close");
close_span.setAttribute("title", "Remove this parameter section.");
label.appendChild(close_span);
default_div.appendChild(label); default_div.appendChild(label);
default_div = render_subform( default_div = render_subform(
json_field.patternProperties['.*'], json_field.patternProperties['.*'],
...@@ -315,6 +321,11 @@ ...@@ -315,6 +321,11 @@
return element; return element;
} }
function removeSubParameter(element) {
$(element).parent().parent().remove();
return false;
}
function addSubForm(element) { function addSubForm(element) {
var subform_json = JSON.parse(atob(element.value)), var subform_json = JSON.parse(atob(element.value)),
input_text = element.parentNode.querySelector("input[type='text']"), input_text = element.parentNode.querySelector("input[type='text']"),
...@@ -344,6 +355,7 @@ ...@@ -344,6 +355,7 @@
field_list = g.props.element.querySelectorAll(".slapos-parameter"), field_list = g.props.element.querySelectorAll(".slapos-parameter"),
button_list = g.props.element.querySelectorAll('button.add-sub-form'), button_list = g.props.element.querySelectorAll('button.add-sub-form'),
label_list = g.props.element.querySelectorAll('label.slapos-parameter-dict-key'), label_list = g.props.element.querySelectorAll('label.slapos-parameter-dict-key'),
close_list = g.props.element.querySelectorAll(".bt_close"),
i, i,
promise_list = []; promise_list = [];
...@@ -374,6 +386,15 @@ ...@@ -374,6 +386,15 @@
)); ));
} }
for (i = 0; i < close_list.length; i = i + 1) {
promise_list.push(loopEventListener(
close_list[i],
'click',
false,
removeSubParameter.bind(g, close_list [i])
));
}
return RSVP.all(promise_list); return RSVP.all(promise_list);
} }
......
master/bt5/slapos_web/PathTemplateItem/web_page_module/rjs_slapos_parameter_form_js.xml
View file @ dc6e39cb
...@@ -236,7 +236,7 @@ ...@@ -236,7 +236,7 @@
</item> </item>
<item> <item>
<key> <string>serial</string> </key> <key> <string>serial</string> </key>
<value> <string>950.63459.41632.30105</string> </value> <value> <string>954.11326.56915.27153</string> </value>
</item> </item>
<item> <item>
<key> <string>state</string> </key> <key> <string>state</string> </key>
...@@ -254,7 +254,7 @@ ...@@ -254,7 +254,7 @@
</tuple> </tuple>
<state> <state>
<tuple> <tuple>
<float>1462374087.76</float> <float>1474892353.46</float>
<string>UTC</string> <string>UTC</string>
</tuple> </tuple>
</state> </state>
......
master/bt5/slapos_web/SkinTemplateItem/portal_skins/vifib_jauks_theme/vifib_style/vifib.css.zpt
View file @ dc6e39cb
...@@ -325,6 +325,21 @@ fieldset > .subfield > label { ...@@ -325,6 +325,21 @@ fieldset > .subfield > label {
text-decoration: none; text-decoration: none;
} }
.bt_close, .subfield .slapos-parameter-dict-key span.bt_close{
padding: 0 6px;
display: block;
float: right;
text-overflow:clip;
white-space:nowrap;
overflow: hidden;
font-size: 1.5em;
border-radius: 2px;
}
.bt_close:hover {
background: #81afab;
color: #fff;
}
.hs-short-title{ .hs-short-title{
margin-left:6px; margin-left:6px;
padding-bottom: 10px; padding-bottom: 10px;
......
master/product/Vifib/Tool/SlapTool.py
View file @ dc6e39cb
...@@ -518,6 +518,18 @@ class SlapTool(BaseTool): ...@@ -518,6 +518,18 @@ class SlapTool(BaseTool):
connection_xml, connection_xml,
slave_reference) slave_reference)
security.declareProtected(Permissions.AccessContentsInformation,
'updateComputerPartitionRelatedInstanceList')
def updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id,
instance_reference_xml):
"""
Update Software Instance predecessor list
"""
return self._updateComputerPartitionRelatedInstanceList(computer_id,
computer_partition_id,
instance_reference_xml)
security.declareProtected(Permissions.AccessContentsInformation, security.declareProtected(Permissions.AccessContentsInformation,
'supplySupply') 'supplySupply')
def supplySupply(self, url, computer_id, state='available'): def supplySupply(self, url, computer_id, state='available'):
...@@ -1365,6 +1377,44 @@ class SlapTool(BaseTool): ...@@ -1365,6 +1377,44 @@ class SlapTool(BaseTool):
software_instance._instance_guid = instance_guid software_instance._instance_guid = instance_guid
return xml_marshaller.xml_marshaller.dumps(software_instance) return xml_marshaller.xml_marshaller.dumps(software_instance)
@UnrestrictedMethod
def _updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id, instance_reference_xml):
"""
Update Software Instance predecessor list to match the given list. If one
instance was not requested by this computer partition, it should be removed
in the predecessor_list of this instance.
Once the link is removed, this instance will be trashed by Garbage Collect!
instance_reference_xml contain list of title of sub-instances requested by
this instance.
"""
software_instance_document = self.\
_getSoftwareInstanceForComputerPartition(computer_id,
computer_partition_id)
cache_reference = '%s-PREDLIST' % software_instance_document.getReference()
if self._getLastData(cache_reference) != instance_reference_xml:
instance_reference_list = xml_marshaller.xml_marshaller.loads(
instance_reference_xml)
current_predecessor_list = software_instance_document.getPredecessorValueList(
portal_type=['Software Instance', 'Slave Instance'])
current_predecessor_title_list = [i.getTitle() for i in
current_predecessor_list]
# If there are items to remove
if list(set(current_predecessor_title_list).difference(instance_reference_list)) != []:
predecessor_list = [instance.getRelativeUrl() for instance in
current_predecessor_list if instance.getTitle()
in instance_reference_list]
LOG('SlapTool', INFO, '%s, %s: Updating predecessor list to %s' % (
computer_id, computer_partition_id, predecessor_list), error=False)
software_instance_document.edit(predecessor_list=predecessor_list,
comment='predecessor_list edited to unlink non commited instances')
self._storeLastData(cache_reference, instance_reference_xml)
#################################################### ####################################################
# Internals methods # Internals methods
#################################################### ####################################################
......
slapos/grid/slapgrid.py
View file @ dc6e39cb
...@@ -50,6 +50,7 @@ from lxml import etree ...@@ -50,6 +50,7 @@ from lxml import etree
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.slap.slap import ServerError from slapos.slap.slap import ServerError
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
from slapos.util import mkdir_p, chownDirectory, string_to_boolean from slapos.util import mkdir_p, chownDirectory, string_to_boolean
from slapos.grid.exception import BuildoutFailedError from slapos.grid.exception import BuildoutFailedError
from slapos.grid.SlapObject import Software, Partition from slapos.grid.SlapObject import Software, Partition
...@@ -666,6 +667,25 @@ stderr_logfile_backups=1 ...@@ -666,6 +667,25 @@ stderr_logfile_backups=1
if not promise_present: if not promise_present:
self.logger.info("No promise.") self.logger.info("No promise.")
def _endInstallationTransaction(self, computer_partition):
partition_id = computer_partition.getId()
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(self.instance_root,
partition_id,
transaction_file_name)
if os.path.exists(transaction_file_path):
with open(transaction_file_path, 'r') as tf:
try:
computer_partition.setComputerPartitionRelatedInstanceList(
[reference for reference in tf.read().split('\n') if reference]
)
except NotFoundError, e:
# Master doesn't implement this feature ?
self.logger.warning("NotFoundError: %s. \nCannot send requested instance "\
"list to master. Please check if this feature is"\
"implemented on SlapOS Master." % str(e))
def _addFirewallRule(self, rule_command): def _addFirewallRule(self, rule_command):
""" """
""" """
...@@ -904,6 +924,14 @@ stderr_logfile_backups=1 ...@@ -904,6 +924,14 @@ stderr_logfile_backups=1
self.logger.debug('Check if %s requires processing...' % computer_partition_id) self.logger.debug('Check if %s requires processing...' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id) instance_path = os.path.join(self.instance_root, computer_partition_id)
os.environ['SLAPGRID_INSTANCE_ROOT'] = self.instance_root
# Check if transaction file of this partition exists, if the file was created,
# remove it so it will be generate with this new transaction
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % computer_partition_id
transaction_file_path = os.path.join(instance_path, transaction_file_name)
if os.path.exists(transaction_file_path):
os.unlink(transaction_file_path)
# Try to get partition timestamp (last modification date) # Try to get partition timestamp (last modification date)
timestamp_path = os.path.join( timestamp_path = os.path.join(
...@@ -1032,6 +1060,7 @@ stderr_logfile_backups=1 ...@@ -1032,6 +1060,7 @@ stderr_logfile_backups=1
partition_ip_list) partition_ip_list)
self._checkPromises(computer_partition) self._checkPromises(computer_partition)
computer_partition.started() computer_partition.started()
self._endInstallationTransaction(computer_partition)
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE: elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
try: try:
# We want to process the partition, even if stopped, because it should # We want to process the partition, even if stopped, because it should
...@@ -1045,6 +1074,7 @@ stderr_logfile_backups=1 ...@@ -1045,6 +1074,7 @@ stderr_logfile_backups=1
# Instance has to be stopped even if buildout/reporting is wrong. # Instance has to be stopped even if buildout/reporting is wrong.
local_partition.stop() local_partition.stop()
computer_partition.stopped() computer_partition.stopped()
self._endInstallationTransaction(computer_partition)
elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE: elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
local_partition.stop() local_partition.stop()
if self.firewall_conf: if self.firewall_conf:
......
slapos/slap/interface/slap.py
View file @ dc6e39cb
...@@ -337,6 +337,14 @@ class IComputerPartition(IBuildoutController, IRequester): ...@@ -337,6 +337,14 @@ class IComputerPartition(IBuildoutController, IRequester):
computer partition. computer partition.
""" """
def setComputerPartitionRelatedInstanceList(instance_reference_list):
"""
Set relation between this Instance and all his children.
instance_reference_list -- list of instances requested by this Computer
Partition.
"""
class IComputer(Interface): class IComputer(Interface):
""" """
Computer interface specification Computer interface specification
......
slapos/slap/slap.py
View file @ dc6e39cb
...@@ -36,6 +36,7 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease", ...@@ -36,6 +36,7 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease",
"Supply", "OpenOrder", "NotFoundError", "Supply", "OpenOrder", "NotFoundError",
"ResourceNotReady", "ServerError", "ConnectionError"] "ResourceNotReady", "ServerError", "ConnectionError"]
import os
import json import json
import logging import logging
import re import re
...@@ -67,6 +68,7 @@ fallback_logger.addHandler(fallback_handler) ...@@ -67,6 +68,7 @@ fallback_logger.addHandler(fallback_handler)
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance' DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME = '.slapos-request-transaction-%s'
class SlapDocument: class SlapDocument:
def __init__(self, connection_helper=None, hateoas_navigator=None): def __init__(self, connection_helper=None, hateoas_navigator=None):
...@@ -81,6 +83,7 @@ class SlapRequester(SlapDocument): ...@@ -81,6 +83,7 @@ class SlapRequester(SlapDocument):
""" """
Abstract class that allow to factor method for subclasses that use "request()" Abstract class that allow to factor method for subclasses that use "request()"
""" """
def _requestComputerPartition(self, request_dict): def _requestComputerPartition(self, request_dict):
try: try:
xml = self._connection_helper.POST('requestComputerPartition', data=request_dict) xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
...@@ -406,9 +409,38 @@ class ComputerPartition(SlapRequester): ...@@ -406,9 +409,38 @@ class ComputerPartition(SlapRequester):
self._partition_id = partition_id self._partition_id = partition_id
self._request_dict = request_dict self._request_dict = request_dict
# Just create an empty file (for nothing requested yet)
self._updateTransactionFile(partition_reference=None)
def __getinitargs__(self): def __getinitargs__(self):
return (self._computer_id, self._partition_id, ) return (self._computer_id, self._partition_id, )
def _updateTransactionFile(self, partition_reference=None):
"""
Store reference to all Instances requested by this Computer Parition
"""
# Environ variable set by Slapgrid while processing this partition
instance_root = os.environ.get('SLAPGRID_INSTANCE_ROOT', '')
if not instance_root or not self._partition_id:
return
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % self._partition_id
transaction_file_path = os.path.join(instance_root, self._partition_id,
transaction_file_name)
try:
if partition_reference is None:
if os.access(os.path.join(instance_root, self._partition_id), os.W_OK):
if os.path.exists(transaction_file_path):
return
transac_file = open(transaction_file_path, 'w')
transac_file.close()
else:
with open(transaction_file_path, 'a') as transac_file:
transac_file.write('%s\n' % partition_reference)
except OSError:
return
def request(self, software_release, software_type, partition_reference, def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None, shared=False, partition_parameter_kw=None, filter_kw=None,
state=None): state=None):
...@@ -440,6 +472,7 @@ class ComputerPartition(SlapRequester): ...@@ -440,6 +472,7 @@ class ComputerPartition(SlapRequester):
'filter_xml': xml_marshaller.dumps(filter_kw), 'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state), 'state': xml_marshaller.dumps(state),
} }
self._updateTransactionFile(partition_reference)
return self._requestComputerPartition(request_dict) return self._requestComputerPartition(request_dict)
def building(self): def building(self):
...@@ -635,6 +668,15 @@ class ComputerPartition(SlapRequester): ...@@ -635,6 +668,15 @@ class ComputerPartition(SlapRequester):
) )
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
data={
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'instance_reference_xml': xml_marshaller.dumps(instance_reference_list)
}
)
def _addIpv6Brackets(url): def _addIpv6Brackets(url):
# if master_url contains an ipv6 without bracket, add it # if master_url contains an ipv6 without bracket, add it
# Note that this is mostly to limit specific issues with # Note that this is mostly to limit specific issues with
......
slapos/tests/slap.py
View file @ dc6e39cb
...@@ -29,6 +29,7 @@ import logging ...@@ -29,6 +29,7 @@ import logging
import os import os
import unittest import unittest
import urlparse import urlparse
import tempfile
import httmock import httmock
...@@ -53,6 +54,8 @@ class SlapMixin(unittest.TestCase): ...@@ -53,6 +54,8 @@ class SlapMixin(unittest.TestCase):
print 'Testing against SLAP server %r' % self.server_url print 'Testing against SLAP server %r' % self.server_url
self.slap = slapos.slap.slap() self.slap = slapos.slap.slap()
self.partition_id = 'PARTITION_01' self.partition_id = 'PARTITION_01'
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
def tearDown(self): def tearDown(self):
pass pass
...@@ -786,6 +789,84 @@ class TestComputerPartition(SlapMixin): ...@@ -786,6 +789,84 @@ class TestComputerPartition(SlapMixin):
# request was done works correctly # request was done works correctly
self.assertEqual(requested_partition_id, requested_partition.getId()) self.assertEqual(requested_partition_id, requested_partition.getId())
def test_request_with_slapgrid_request_transaction(self):
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
partition_id = 'PARTITION_01'
instance_root = tempfile.mkdtemp()
partition_root = os.path.join(instance_root, partition_id)
os.mkdir(partition_root)
os.environ['SLAPGRID_INSTANCE_ROOT'] = instance_root
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(partition_root, transaction_file_name)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs
and 'computer_partition_reference' in qs):
slap_partition = slapos.slap.ComputerPartition(
qs['computer_reference'][0],
qs['computer_partition_reference'][0])
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
}
elif (url.path == '/getComputerInformation'
and 'computer_id' in qs):
slap_computer = slapos.slap.Computer(qs['computer_id'][0])
slap_computer._software_release_list = []
slap_partition = slapos.slap.ComputerPartition(
qs['computer_id'][0],
partition_id)
slap_computer._computer_partition_list = [slap_partition]
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
}
elif url.path == '/requestComputerPartition':
raise RequestWasCalled
else:
return {
'status_code': 404
}
with httmock.HTTMock(handler):
self.computer_guid = self._getTestComputerId()
self.slap = slapos.slap.slap()
self.slap.initializeConnection(self.server_url)
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content = f.read()
self.assertEqual(content, '')
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'myref')
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Not override
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Request a second instance
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'mysecondref')
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEquals(list(set(content_list)), ['myref', 'mysecondref'])
def _test_new_computer_partition_state(self, state): def _test_new_computer_partition_state(self, state):
""" """
Helper method to automate assertions of failing states on new Computer Helper method to automate assertions of failing states on new Computer
......
slapos/tests/slapgrid.py
View file @ dc6e39cb
...@@ -51,6 +51,7 @@ from slapos.grid.utils import md5digest ...@@ -51,6 +51,7 @@ from slapos.grid.utils import md5digest
from slapos.grid.watchdog import Watchdog from slapos.grid.watchdog import Watchdog
from slapos.grid import SlapObject from slapos.grid import SlapObject
from slapos.grid.SlapObject import WATCHDOG_MARK from slapos.grid.SlapObject import WATCHDOG_MARK
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
import slapos.grid.SlapObject import slapos.grid.SlapObject
import httmock import httmock
...@@ -107,6 +108,8 @@ class BasicMixin(object): ...@@ -107,6 +108,8 @@ class BasicMixin(object):
self._tempdir = tempfile.mkdtemp() self._tempdir = tempfile.mkdtemp()
self.software_root = os.path.join(self._tempdir, 'software') self.software_root = os.path.join(self._tempdir, 'software')
self.instance_root = os.path.join(self._tempdir, 'instance') self.instance_root = os.path.join(self._tempdir, 'instance')
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.setSlapgrid() self.setSlapgrid()
...@@ -338,6 +341,8 @@ class ComputerForTest(object): ...@@ -338,6 +341,8 @@ class ComputerForTest(object):
return {'status_code': 200} return {'status_code': 200}
if url.path == '/softwareInstanceBang': if url.path == '/softwareInstanceBang':
return {'status_code': 200} return {'status_code': 200}
if url.path == "/updateComputerPartitionRelatedInstanceList":
return {'status_code': 200}
if url.path == '/softwareInstanceError': if url.path == '/softwareInstanceError':
instance.error_log = '\n'.join( instance.error_log = '\n'.join(
[ [
...@@ -1647,16 +1652,21 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1647,16 +1652,21 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
# Then run usage report and see if it is still working # Then run usage report and see if it is still working
computer.sequence = [] computer.sequence = []
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# registerComputerPartition will create one more file:
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
...@@ -2313,3 +2323,18 @@ class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase): ...@@ -2313,3 +2323,18 @@ class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase):
rules_list = json.loads(frules.read()) rules_list = json.loads(frules.read())
self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list) self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list)
class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase):
def test_one_partition(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
instance = computer.instance_list[0]
partition = os.path.join(self.instance_root, '0')
request_list_file = os.path.join(partition,
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name)
with open(request_list_file, 'w') as f:
f.write('some partition')
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0'])
self.assertFalse(os.path.exists(request_list_file))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7