slapos_crm: Update SupportRequest_recheckMonitoring

  Update script to handle also stalled cases

master/bt5/slapos_crm/SkinTemplateItem/portal_skins/slapos_crm_monitoring/ComputeNode_checkState.py

@@ -62,8 +62,7 @@ if not should_notify:
 
     if instance_list:
       should_notify = True
-      description = "The Compute Node %s (%s) didnt process its instances for more them 24 hours" % (
-                  compute_node_title, reference)
+      description = "The Compute Node %s (%s) didnt process its instances for more them 24 hours, last contact: %s"
 
     for instance in instance_list:
       instance_access_status = instance.getAccessStatus()
@@ -73,7 +72,7 @@ if not should_notify:
 
       # At lest one partition contacted in the last 24h30min.
       last_contact = max(DateTime(instance_access_status.get('created_at')), last_contact)
-      if (now - DateTime(instance_access_status.get('created_at'))) < 1.01:
+      if (now - DateTime(instance_access_status.get('created_at'))) < 1.05:
         should_notify = False
         description = ""
         break
@@ -91,8 +90,7 @@ if not should_notify:
 
   if software_installation_list:
     should_notify = True
-    description = "The Compute Node %s (%s) didnt process its software releases for more them 24 hours" % (
-                compute_node_title, reference)
+    description = "The Compute Node %s (%s) didnt process its software releases for more them 24 hours, last contact %s"
 
   # Test if server didnt process the internal softwares releases for more them 24h
   for installation in software_installation_list:
@@ -119,7 +117,7 @@ if should_notify:
 
   if support_request is None:
     person.notify(support_request_title=ticket_title,
-                support_request_description=description,
+      support_request_description=description % (context.getTitle(), reference, last_contact),
       aggregate=context.getRelativeUrl())
 
     support_request_relative_url = context.REQUEST.get("support_request_relative_url")
@@ -136,7 +134,7 @@ if should_notify:
     reference=notification_message_reference)
 
   if notification_message is None:
-    message = """%s""" % description
+    message = """%s""" % (description % (context.getTitle(), reference, last_contact))
   else:
     mapping_dict = {'compute_node_title':context.getTitle(),
                     'compute_node_id':reference,

master/bt5/slapos_crm/SkinTemplateItem/portal_skins/slapos_crm_monitoring/SupportRequest_recheckMonitoring.py

@@ -13,6 +13,9 @@ if context.getSimulationState() == "invalidated":
 if context.getPortalType() != "Support Request":
   return "Not a Support Request"
 
+
+now = DateTime()
+portal = context.getPortalObject()
 document = context.getAggregateValue()
 if document is None:
   return True
@@ -27,7 +30,57 @@ if aggregate_portal_type == "Compute Node":
     return "No Contact Information"
 
   last_contact = DateTime(d.get('created_at'))
-  if (DateTime() - last_contact) < 0.01:
+  if (now - last_contact) < 0.01:
+    # If server has no partitions skip
+    compute_partition_uid_list = [
+      x.getUid() for x in document.contentValues(portal_type="Compute Partition")
+      if x.getSlapState() == 'busy']
+
+    if compute_partition_uid_list:
+      is_instance_stalled = True
+      last_contact = None
+      instance_list = portal.portal_catalog(
+        portal_type='Software Instance',
+        default_aggregate_uid=compute_partition_uid_list)
+
+      for instance in instance_list:
+        instance_access_status = instance.getAccessStatus()
+        if instance_access_status.get('no_data', None):
+          # Ignore if there isnt any data
+          continue
+
+        # At lest one partition contacted in the last 24h30min.
+        last_contact = max(DateTime(instance_access_status.get('created_at')), last_contact)
+        if (now - DateTime(instance_access_status.get('created_at'))) < 1.05:
+          is_instance_stalled = False
+          break
+
+      if is_instance_stalled and len(instance_list):
+        return "Process instance stalled, last contact was %s" % last_contact
+
+    # Since server is contacting, check for stalled software releases processes
+    is_software_stalled = True
+    last_contact = None
+    software_installation_list = portal.portal_catalog(
+      portal_type='Software Installation',
+      default_aggregate_uid=document.getUid(),
+      validation_state='validated')
+
+    # Test if server didnt process the internal softwares releases for more them 24h
+    for installation in software_installation_list:
+      installation_access_status = installation.getAccessStatus()
+      if installation_access_status.get('no_data', None):
+        # Ignore if there isnt any data on it
+        continue
+
+      last_contact = max(DateTime(installation_access_status.get('created_at')), last_contact)
+      if (now - DateTime(installation_access_status.get('created_at'))) < 1.01:
+        is_software_stalled = False
+        break
+
+    if is_software_stalled and len(software_installation_list):
+      return "Process instance stalled, last contact was %s" % last_contact
+
     return "All OK, latest contact: %s " % last_contact
   else:
     return "Problem, latest contact: %s" % last_contact
@@ -40,7 +93,7 @@ if aggregate_portal_type == "Software Installation":
   if document.getSlapState() not in ["start_requested", "stop_requested"]:
     return "Software Installation is Destroyed."
 
-  d = context.getAccessStatus()
+  d = document.getAccessStatus()
   if d.get("no_data", None) == 1:
     return "The software release %s did not started to build on %s since %s" % \
         (document.getUrlString(), compute_node_title, document.getCreationDate())