Commit e36b68d4 authored by Rafael Monnerat's avatar Rafael Monnerat

slapos_slap_tool: Cache the XML to preserve performance

  Re-serialise the object from the dict after hit cache is slow enough
to cause problems ounder high load.

  This change preserve the previous behaviour reaching the final XML
from cache keeping performace on SlapTool and allowing new
implementation benefit from cache at same time.

  Fixup test to match xml with xml.
parent 0dbf8e50
...@@ -109,11 +109,14 @@ class SlapOSComputeNodeMixin(object): ...@@ -109,11 +109,14 @@ class SlapOSComputeNodeMixin(object):
def _fillComputeNodeInformationCache(self, user): def _fillComputeNodeInformationCache(self, user):
key = '%s_%s' % (self.getReference(), user) key = '%s_%s' % (self.getReference(), user)
try: try:
computer_dict = self._getCacheComputeNodeInformation(user)
self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE, self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
dict ( dict (
time=time.time(), time=time.time(),
refresh_etag=self._calculateRefreshEtag(), refresh_etag=self._calculateRefreshEtag(),
data=self._getCacheComputeNodeInformation(user), data=computer_dict,
# Store the XML while SlapTool Still used
data_xml=self.getPortalObject().portal_slap._getSlapComputeNodeXMLFromDict(computer_dict)
), ),
cache_duration=self.getPortalObject().portal_caches\ cache_duration=self.getPortalObject().portal_caches\
.getRamCacheRoot().get('compute_node_information_cache_factory'\ .getRamCacheRoot().get('compute_node_information_cache_factory'\
...@@ -154,7 +157,6 @@ class SlapOSComputeNodeMixin(object): ...@@ -154,7 +157,6 @@ class SlapOSComputeNodeMixin(object):
user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue( user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
reference=user, portal_type=['Person', 'Compute Node', 'Software Instance'])) reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
user_type = user_document.getPortalType() user_type = user_document.getPortalType()
self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
if user_type in ('Compute Node', 'Person'): if user_type in ('Compute Node', 'Person'):
if not self._isTestRun(): if not self._isTestRun():
...@@ -191,7 +193,8 @@ class SlapOSComputeNodeMixin(object): ...@@ -191,7 +193,8 @@ class SlapOSComputeNodeMixin(object):
portal_type="Compute Partition", portal_type="Compute Partition",
checked_permission="View") checked_permission="View")
else: else:
compute_partition_list = self.getPortalObject().portal_catalog.unrestrictedSearchResults( unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
compute_partition_list = unrestrictedSearchResults(
parent_uid=self.getUid(), parent_uid=self.getUid(),
validation_state="validated", validation_state="validated",
portal_type="Compute Partition") portal_type="Compute Partition")
......
...@@ -88,8 +88,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin): ...@@ -88,8 +88,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.commit() self.commit()
first_etag = self.compute_node._calculateRefreshEtag() first_etag = self.compute_node._calculateRefreshEtag()
first_body_fingerprint = hashData( first_body_fingerprint = hashData(
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id) self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
) )
)
self.assertEqual(200, response.status) self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers) self.assertTrue('last-modified' not in response.headers)
self.assertEqual(first_etag, response.headers.get('etag')) self.assertEqual(first_etag, response.headers.get('etag'))
...@@ -123,8 +125,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin): ...@@ -123,8 +125,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.assertTrue('last-modified' not in response.headers) self.assertTrue('last-modified' not in response.headers)
second_etag = self.compute_node._calculateRefreshEtag() second_etag = self.compute_node._calculateRefreshEtag()
second_body_fingerprint = hashData( second_body_fingerprint = hashData(
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id) self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
) )
)
self.assertNotEqual(first_etag, second_etag) self.assertNotEqual(first_etag, second_etag)
# The indexation timestamp does not impact the response body # The indexation timestamp does not impact the response body
self.assertEqual(first_body_fingerprint, second_body_fingerprint) self.assertEqual(first_body_fingerprint, second_body_fingerprint)
...@@ -156,8 +160,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin): ...@@ -156,8 +160,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
# Edition does not impact the etag # Edition does not impact the etag
self.assertEqual(second_etag, self.compute_node._calculateRefreshEtag()) self.assertEqual(second_etag, self.compute_node._calculateRefreshEtag())
third_body_fingerprint = hashData( third_body_fingerprint = hashData(
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id) self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
) )
)
# The edition impacts the response body # The edition impacts the response body
self.assertNotEqual(first_body_fingerprint, third_body_fingerprint) self.assertNotEqual(first_body_fingerprint, third_body_fingerprint)
response = self.portal_slap.getFullComputerInformation(self.compute_node_id) response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
...@@ -195,7 +201,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin): ...@@ -195,7 +201,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
# The edition does not impact the response body yet, as the aggregate relation # The edition does not impact the response body yet, as the aggregate relation
# is not yet unindex # is not yet unindex
self.assertEqual(third_body_fingerprint, hashData( self.assertEqual(third_body_fingerprint, hashData(
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id) self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
)) ))
response = self.portal_slap.getFullComputerInformation(self.compute_node_id) response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
self.commit() self.commit()
...@@ -217,8 +225,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin): ...@@ -217,8 +225,10 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.assertTrue('last-modified' not in response.headers) self.assertTrue('last-modified' not in response.headers)
fourth_etag = self.compute_node._calculateRefreshEtag() fourth_etag = self.compute_node._calculateRefreshEtag()
fourth_body_fingerprint = hashData( fourth_body_fingerprint = hashData(
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id) self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
) )
)
self.assertNotEqual(third_etag, fourth_etag) self.assertNotEqual(third_etag, fourth_etag)
# The indexation timestamp does not impact the response body # The indexation timestamp does not impact the response body
self.assertNotEqual(third_body_fingerprint, fourth_body_fingerprint) self.assertNotEqual(third_body_fingerprint, fourth_body_fingerprint)
......
...@@ -39,6 +39,8 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool ...@@ -39,6 +39,8 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions from Products.ERP5Type import Permissions
from Products.ERP5Type.Cache import CachingMethod from Products.ERP5Type.Cache import CachingMethod
from erp5.component.module.SlapOSCloud import _assertACI from erp5.component.module.SlapOSCloud import _assertACI
from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
from lxml import etree from lxml import etree
try: try:
from slapos.slap.slap import ( from slapos.slap.slap import (
...@@ -169,60 +171,56 @@ class SlapTool(BaseTool): ...@@ -169,60 +171,56 @@ class SlapTool(BaseTool):
Reuses slap library for easy marshalling. Reuses slap library for easy marshalling.
""" """
user = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserName() portal = self.getPortalObject()
user = portal.portal_membership.getAuthenticatedMember().getUserName()
if str(user) == computer_id: if str(user) == computer_id:
compute_node = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserValue() compute_node = portal.portal_membership.getAuthenticatedMember().getUserValue()
compute_node.setAccessStatus(computer_id) compute_node.setAccessStatus(computer_id)
else: else:
# Don't use getDocument because we don't want use _assertACI here, but # Don't use getDocument because we don't want use _assertACI here, but
# just call the API on computer. # just call the API on computer.
compute_node = self.getPortalObject().portal_catalog.unrestrictedSearchResults( compute_node = portal.portal_catalog.unrestrictedSearchResults(
portal_type='Compute Node', reference=computer_id, portal_type='Compute Node', reference=computer_id,
validation_state="validated")[0].getObject() validation_state="validated")[0].getObject()
refresh_etag = compute_node._calculateRefreshEtag() refresh_etag = compute_node._calculateRefreshEtag()
computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
## Horrible code starts # Keep compatibility with older clients that relies on marshalling.
# Convert computer node into SlapComputer # This code is an adaptation of SlapOSComputeNodeMixin._getComputeNodeInformation
slap_compute_node = ComputeNode(computer_dict["_computer_id"]) # To comply with cache capability.
slap_compute_node._computer_partition_list = [] user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
slap_compute_node._software_release_list = [] reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
user_type = user_document.getPortalType()
for partition_dict in computer_dict["_computer_partition_list"]: if user_type in ('Compute Node', 'Person'):
slap_compute_partition = SlapComputePartition( if not compute_node._isTestRun():
partition_id=partition_dict["partition_id"], cache_plugin = compute_node._getCachePlugin()
computer_id=partition_dict['compute_node_id'] key = '%s_%s' % (compute_node.getReference(), user)
) try:
slap_compute_partition._requested_state = partition_dict["_requested_state"] entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
slap_compute_partition._need_modification = partition_dict["_need_modification"] except KeyError:
if partition_dict["_software_release_document"] is not None: entry = None
slap_compute_partition._access_status = partition_dict["_access_status"]
slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"] if entry is not None and isinstance(entry.getValue(), dict):
slap_compute_partition._connection_dict = partition_dict["_connection_dict"] cached_dict = entry.getValue()
slap_compute_partition._filter_dict = partition_dict["_filter_dict"] cached_etag = cached_dict.get('refresh_etag', None)
slap_compute_partition._instance_guid = partition_dict["_instance_guid"] if (refresh_etag != cached_etag):
slap_compute_partition._software_release_document = SoftwareRelease( # Do not recalculate the compute_node information
software_release=partition_dict["_software_release_document"]["software_release"], # if nothing changed
computer_guid=partition_dict["_software_release_document"]["computer_guid"]) compute_node._activateFillComputeNodeInformationCache(user)
etag = cached_etag
body = cached_dict['data_xml']
else: else:
slap_compute_partition._software_release_document = None compute_node._activateFillComputeNodeInformationCache(user)
self.REQUEST.response.setStatus(503)
slap_compute_node._computer_partition_list.append( return self.REQUEST.response
slap_compute_partition else:
) computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
body = self._getSlapComputeNodeXMLFromDict(computer_dict)
for software_release_dict in computer_dict['_software_release_list']: else:
slap_software_release = SoftwareRelease( computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
software_release=software_release_dict["software_release"], body = self._getSlapComputeNodeXMLFromDict(computer_dict)
computer_guid=software_release_dict['computer_guid'])
slap_software_release._requested_state = software_release_dict['_requested_state']
slap_software_release._known_state = software_release_dict['_known_state']
slap_compute_node._software_release_list.append(slap_software_release)
body = dumps(slap_compute_node)
## Horrible code ends
self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
if self.REQUEST.response.getStatus() == 200: if self.REQUEST.response.getStatus() == 200:
# Keep in cache server for 7 days # Keep in cache server for 7 days
self.REQUEST.response.setHeader('Cache-Control', self.REQUEST.response.setHeader('Cache-Control',
...@@ -734,7 +732,6 @@ class SlapTool(BaseTool): ...@@ -734,7 +732,6 @@ class SlapTool(BaseTool):
#################################################### ####################################################
# Internal methods # Internal methods
#################################################### ####################################################
def _validateXML(self, to_be_validated, xsd_model): def _validateXML(self, to_be_validated, xsd_model):
"""Will validate the xml file""" """Will validate the xml file"""
#We parse the XSD model #We parse the XSD model
...@@ -1012,6 +1009,44 @@ class SlapTool(BaseTool): ...@@ -1012,6 +1009,44 @@ class SlapTool(BaseTool):
# Internals methods # Internals methods
#################################################### ####################################################
def _getSlapComputeNodeXMLFromDict(self, computer_dict):
slap_compute_node = ComputeNode(computer_dict["_computer_id"])
slap_compute_node._computer_partition_list = []
slap_compute_node._software_release_list = []
for partition_dict in computer_dict["_computer_partition_list"]:
slap_compute_partition = SlapComputePartition(
partition_id=partition_dict["partition_id"],
computer_id=partition_dict['compute_node_id']
)
slap_compute_partition._requested_state = partition_dict["_requested_state"]
slap_compute_partition._need_modification = partition_dict["_need_modification"]
if partition_dict["_software_release_document"] is not None:
slap_compute_partition._access_status = partition_dict["_access_status"]
slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"]
slap_compute_partition._connection_dict = partition_dict["_connection_dict"]
slap_compute_partition._filter_dict = partition_dict["_filter_dict"]
slap_compute_partition._instance_guid = partition_dict["_instance_guid"]
slap_compute_partition._software_release_document = SoftwareRelease(
software_release=partition_dict["_software_release_document"]["software_release"],
computer_guid=partition_dict["_software_release_document"]["computer_guid"])
else:
slap_compute_partition._software_release_document = None
slap_compute_node._computer_partition_list.append(
slap_compute_partition
)
for software_release_dict in computer_dict['_software_release_list']:
slap_software_release = SoftwareRelease(
software_release=software_release_dict["software_release"],
computer_guid=software_release_dict['computer_guid'])
slap_software_release._requested_state = software_release_dict['_requested_state']
slap_software_release._known_state = software_release_dict['_known_state']
slap_compute_node._software_release_list.append(slap_software_release)
return dumps(slap_compute_node)
def _getSoftwareInstanceForComputePartition(self, compute_node_id, def _getSoftwareInstanceForComputePartition(self, compute_node_id,
compute_partition_id, slave_reference=None): compute_partition_id, slave_reference=None):
compute_partition_document = self.getPortalObject().portal_catalog.getComputePartitionObject( compute_partition_document = self.getPortalObject().portal_catalog.getComputePartitionObject(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment