...
 
Commits (39)
Showing 98 changed files with 947 additions and 1268 deletions
......@@ -88,6 +88,7 @@
<string>my_source_function</string>
<string>my_source_funding</string>
<string>my_source_project_title</string>
<string>my_source_payment_request_title</string>
<string>my_source_payment</string>
<string>my_payment_mode</string>
<string>my_aggregate_title_list</string>
......
......@@ -88,6 +88,7 @@
<string>my_destination_function</string>
<string>my_destination_funding</string>
<string>my_destination_project_title</string>
<string>my_destination_payment_request_title</string>
<string>my_destination_payment</string>
<string>my_payment_mode</string>
<string>my_aggregate_title_list</string>
......
......@@ -80,6 +80,7 @@
<string>my_destination_project_title</string>
<string>my_destination_reference</string>
<string>my_destination_carrier_title</string>
<string>my_destination_payment_request_title</string>
</list>
</value>
</item>
......@@ -95,6 +96,7 @@
<string>my_source_project_title</string>
<string>my_source_reference</string>
<string>my_source_carrier_title</string>
<string>my_source_payment_request_title</string>
</list>
</value>
</item>
......
......@@ -80,6 +80,7 @@
<string>my_source_project_title</string>
<string>my_source_reference</string>
<string>my_source_carrier_title</string>
<string>my_source_payment_request_title</string>
</list>
</value>
</item>
......@@ -95,6 +96,7 @@
<string>my_destination_project_title</string>
<string>my_destination_reference</string>
<string>my_destination_carrier_title</string>
<string>my_destination_payment_request_title</string>
</list>
</value>
</item>
......
......@@ -48,6 +48,13 @@
if (site) {
return gadget.redirect({ command: "row", url: site});
}
// User entered wrong password ?
// Notify
return gadget.notifySubmitted({message: 'Unauthorized storage access', status: 'error'})
.push(function () {
return gadget.redirect({command: 'display',
options: {page: 'ojs_configurator'}});
});
}
throw error;
});
......@@ -60,6 +67,7 @@
gadget.state_parameter_dict = {};
})
.declareAcquiredMethod("notifySubmitted", "notifySubmitted")
.declareAcquiredMethod("redirect", "redirect")
.declareAcquiredMethod("getSetting", "getSetting")
.declareAcquiredMethod("setSetting", "setSetting")
......
......@@ -220,7 +220,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>vincent</string> </value>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -234,7 +234,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>962.36985.29761.10359</string> </value>
<value> <string>973.43582.44368.54391</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -252,7 +252,7 @@
</tuple>
<state>
<tuple>
<float>1507216488.77</float>
<float>1550152243.11</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -13,7 +13,16 @@
})
.push(function (setting) {
var configuration = {},
attachment_synchro = setting[1] !== "";
attachment_synchro = setting[1] !== "",
linshare_json = {
type: "linshare",
url: options.url
};
if (options.username || options.password) {
linshare_json.access_token = window.btoa(
options.username + ':' + options.password
);
}
configuration = {
type: "replicate",
query: {
......@@ -69,13 +78,7 @@
},
sub_storage: {
type: "query",
sub_storage: {
type: "linshare",
url: options.url,
access_token: window.btoa(
options.username + ':' + options.password
)
}
sub_storage: linshare_json
}
}
}
......@@ -111,7 +114,8 @@
var gadget = this;
if (options.url) {
return gadget.changeState({
url: options.url || ""
url: options.url || "",
username: options.username || ""
});
}
return gadget.getSetting('linshare_storage', "")
......@@ -160,7 +164,7 @@
"my_username": {
"description": "",
"title": "Username",
"default": "",
"default": gadget.state.username || "",
"css_class": "",
"required": 1,
"editable": 1,
......
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>973.24661.29279.57753</string> </value>
<value> <string>973.43371.54750.44595</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1549019475.4</float>
<float>1550139619.39</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -22,7 +22,7 @@
</div>
</div>
<div class="panel_img">
<img class="ui-title" alt="OfficeJS" src="officejs_logo.png?format=png"/>
<img class="ui-title" alt="OfficeJS" src="gadget_erp5_panel.png?format=png"/>
</div>
</div>
</script>
......
......@@ -261,7 +261,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>969.52327.64978.40960</string> </value>
<value> <string>973.40404.23731.64460</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -279,7 +279,7 @@
</tuple>
<state>
<tuple>
<float>1535121191.77</float>
<float>1549966400.8</float>
<string>UTC</string>
</tuple>
</state>
......
erp5_officejs
erp5_web_renderjs_ui
erp5_software_pdm
\ No newline at end of file
......@@ -166,11 +166,7 @@ appstore/dev/index.en.html\n
appstore/dev/index.html\n
appstore/documentation/brainstorming_page_json.txt\n
appstore/documentation/documentation.txt\n
appstore/img/appjabbericon.svg\n
appstore/img/erp5-logo.png\n
appstore/img/flags.png\n
appstore/img/slapos.png\n
appstore/img/vifib-logo.png\n
appstore/\n
appstore/js/bin.js\n
appstore/js/erp5_loader.js\n
......@@ -340,7 +336,7 @@ NETWORK:\n
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>973.19272.23311.6092</string> </value>
<value> <string>973.40487.50916.35840</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -358,7 +354,7 @@ NETWORK:\n
</tuple>
<state>
<tuple>
<float>1548694049.98</float>
<float>1549966616.29</float>
<string>UTC</string>
</tuple>
</state>
......
"""Save the message id of the relative document"""
if document_relative_url:
document = context.getPortalObject().restrictedTraverse(document_relative_url)
document.edit(destination_reference=message_id_list[0],
gateway = gateway_relative_url)
document.edit(destination_reference=message_id,
gateway=gateway_relative_url)
......@@ -50,7 +50,7 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>message_id_list, document_relative_url=None, gateway_relative_url=None, **kw</string> </value>
<value> <string>message_id, document_relative_url=None, gateway_relative_url=None, **kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
......
......@@ -4,42 +4,29 @@
"""
#Get recipients
if not to_url:
recipient_phone_list = [person.getDefaultMobileTelephoneValue() for person in context.getDestinationValueList()]
if None in recipient_phone_list:
raise ValueError("All recipients should have a default mobile phone")
recipient_phone_list = [
person.getDefaultMobileTelephoneValue() for person in context.getDestinationValueList()]
if None in recipient_phone_list:
raise ValueError("All recipients should have a default mobile phone")
to_url = [phone.asURL() for phone in recipient_phone_list]
if None in to_url:
raise ValueError("All recipients should have a valid default mobile phone number")
to_url = [phone.asURL() for phone in recipient_phone_list]
if None in to_url:
raise ValueError("All recipients should have a valid default mobile phone number")
#Get sender
if not from_url:
if context.getSourceValue():
sender_phone = context.getSourceValue().getDefaultMobileTelephoneValue()
if not sender_phone:
raise ValueError("The sender(%s) should have a default mobile phone" % context.getSourceValue())
#We use title of sender
from_title = sender_phone.getTitle()
from_url = sender_phone.asURL()
if not body:
body = context.getTextContent()
body = context.getTextContent()
if not context.getStartDate():
context.setStartDate(DateTime())
context.portal_sms.activate(
activity="SQLQueue",
# We do not retry these activities not to send SMS multiple times
max_retry=0,
conflict_retry=False,
).send(
text=body,
recipient=to_url,
sender=from_url,
sender_title=from_title,
message_type="text",
test=download,
document_relative_url=context.getRelativeUrl(),
**kw)
for recipient in context.getDestinationList():
context.portal_sms.activate(
activity="SQLQueue",
# We do not retry these activities not to send SMS multiple times
max_retry=0,
conflict_retry=False,
).send(
text=body,
sender=context.getSource(),
recipient=recipient,
document_relative_url=context.getRelativeUrl(),
)
......@@ -50,7 +50,7 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>from_url=None, from_title=None, to_url=None, reply_url=None, subject=None, body=None, attachment_format=None, attachment_list=None,download=False,**kw</string> </value>
<value> <string>from_url=None, from_title=None, to_url=None, reply_url=None, subject=None, body=None, attachment_format=None, attachment_list=None, download=False, **kw</string> </value>
</item>
<item>
<key> <string>_proxy_roles</string> </key>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SMS Tool" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_Transient_Objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Access_future_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Access_session_data_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_folders_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Change_local_roles_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Copy_or_Move_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Delete_objects_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_FTP_access_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_List_folder_contents_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_List_portal_members_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Manage_properties_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Reply_to_item_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Review_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Search_ZCatalog_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Set_own_properties_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Undo_changes_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Use_mailhost_services_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_History_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Author</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignor</string>
<string>Associate</string>
<string>Assignee</string>
<string>Manager</string>
<string>Auditor</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_management_screens_Permission</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>portal_sms</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
portal_sms
\ No newline at end of file
......@@ -148,6 +148,8 @@ gadget_hr_translation_data.js\n
gadget_officejs_widget_listbox.js\n
erp5_launcher.js\n
erp5_launcher.html\n
gadget_officejs_hr_web_manifest.json\n
hr_logo.svg\n
\n
gadget_officejs_page_expense_report.html\n
gadget_officejs_page_expense_report.js\n
......@@ -380,7 +382,7 @@ NETWORK:\n
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>966.59986.993.48708</string> </value>
<value> <string>973.40447.29564.60893</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -398,8 +400,8 @@ NETWORK:\n
</tuple>
<state>
<tuple>
<float>1523958492.04</float>
<string>GMT+2</string>
<float>1549964903.16</float>
<string>UTC</string>
</tuple>
</state>
</object>
......
......@@ -10,6 +10,7 @@
<link href="font-awesome/font-awesome.css" rel="stylesheet" type="text/css" />
<link rel="stylesheet" href="jquerymobile.css">
<link rel="stylesheet" href="gadget_erp5.css">
${webapp_manifest_full_link_tag}
<script data-renderjs-configuration="application_title" type="text/x-renderjs-configuration">${application_title}</script>
<script data-renderjs-configuration="panel_gadget" type="text/x-renderjs-configuration">${panel_gadget}</script>
......
......@@ -222,7 +222,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>cedric.le.ninivin</string> </value>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -236,7 +236,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>962.4234.59641.41216</string> </value>
<value> <string>973.40447.29564.60893</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -254,7 +254,7 @@
</tuple>
<state>
<tuple>
<float>1505315239.43</float>
<float>1549964847.35</float>
<string>UTC</string>
</tuple>
</state>
......
{
"short_name": "HR",
"name": "OfficeJS HR",
"description": "Business travel, Expense remboursement, Leave request management",
"icons": [{
"src": "hr_logo.svg",
"sizes": "any",
"type": "image/svg"
}],
"start_url": "../../",
"display": "standalone"
}
\ No newline at end of file
......@@ -299,6 +299,16 @@
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>configuration_webapp_manifest_url</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
......@@ -386,6 +396,10 @@
<value> <string>object_view</string> </value>
</item>
<item>
<key> <string>configuration_webapp_manifest_url</string> </key>
<value> <string>gadget_officejs_hr_web_manifest.json</string> </value>
</item>
<item>
<key> <string>configuration_x_frame_options</string> </key>
<value> <string>SAMEORIGIN</string> </value>
</item>
......@@ -624,7 +638,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>966.51303.25888.32000</string> </value>
<value> <string>973.40447.29564.60893</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -642,8 +656,8 @@
</tuple>
<state>
<tuple>
<float>1523518073.81</float>
<string>GMT+2</string>
<float>1549964409.02</float>
<string>UTC</string>
</tuple>
</state>
</object>
......
......@@ -13111,7 +13111,12 @@ return new Parser;
_linshare_uuid: entry_list[i].uuid
};
if (options.include_docs === true) {
entry.doc = JSON.parse(entry_list[i].metaData) || {};
try {
entry.doc = JSON.parse(entry_list[i].metaData) || {};
} catch (error) {
// Metadata are not always JSON
entry.doc = {};
}
}
result_list.push(entry);
......
......@@ -234,7 +234,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>973.4826.12688.30276</string> </value>
<value> <string>973.43645.19782.57207</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -252,7 +252,7 @@
</tuple>
<state>
<tuple>
<float>1548090618.17</float>
<float>1550160394.98</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -9,6 +9,9 @@ file_content = file.getData()
# The vanilla HTML is wanted
response.setBase(None)
# Allow any external app to download the source code
response.setHeader("Access-Control-Allow-Origin", "*")
if REQUEST.getHeader('If-Modified-Since', '') == file.getModificationDate().rfc822():
response.setStatus(304)
return ""
......
......@@ -6,6 +6,9 @@ if response is None:
# The vanilla HTML is wanted
response.setBase(None)
# Allow any external app to download the source code
response.setHeader("Access-Control-Allow-Origin", "*")
image = context
if REQUEST.getHeader('If-Modified-Since', '') == image.getModificationDate().rfc822():
response.setStatus(304)
......
......@@ -6,6 +6,9 @@ if response is None:
# The vanilla HTML is wanted
response.setBase(None)
# Allow any external app to download the source code
response.setHeader("Access-Control-Allow-Origin", "*")
web_page = context
web_section = REQUEST.get("current_web_section")
if web_section is None:
......
......@@ -33,14 +33,6 @@ from zLOG import LOG, WARNING, ERROR
from ZODB.POSException import ConflictError
from cStringIO import StringIO
import transaction
# Error values for message validation
EXCEPTION = -1
VALID = 0
INVALID_PATH = 1
INVALID_ORDER = 2
# Time global parameters
MAX_PROCESSING_TIME = 900 # in seconds
VALIDATION_ERROR_DELAY = 15 # in seconds
......@@ -96,52 +88,6 @@ class Queue(object):
def distribute(self, activity_tool, node_count):
raise NotImplementedError
def validate(self, activity_tool, message, check_order_validation=1, **kw):
"""
This is the place where activity semantics is implemented
**kw contains all parameters which allow to implement synchronisation,
constraints, delays, etc.
Standard synchronisation parameters:
after_method_id -- never validate message if after_method_id
is in the list of methods which are
going to be executed
after_message_uid -- never validate message if after_message_uid
is in the list of messages which are
going to be executed
after_path -- never validate message if after_path
is in the list of path which are
going to be executed
"""
try:
if activity_tool.unrestrictedTraverse(message.object_path, None) is None:
# Do not try to call methods on objects which do not exist
LOG('CMFActivity', WARNING,
'Object %s does not exist' % '/'.join(message.object_path))
return INVALID_PATH
if check_order_validation:
for k, v in kw.iteritems():
if activity_tool.validateOrder(message, k, v):
return INVALID_ORDER
except ConflictError:
raise
except:
LOG('CMFActivity', WARNING,
'Validation of Object %s raised exception' % '/'.join(message.object_path),
error=sys.exc_info())
# Do not try to call methods on objects which cause errors
return EXCEPTION
return VALID
def getDependentMessageList(self, activity_tool, message):
message_list = []
for k, v in message.activity_kw.iteritems():
message_list += activity_tool.getDependentMessageList(message, k, v)
return message_list
def getExecutableMessageList(self, activity_tool, message, message_dict,
validation_text_dict, now_date=None):
"""Get messages which have no dependent message, and store them in the dictionary.
......@@ -165,8 +111,7 @@ class Queue(object):
cached_result = validation_text_dict.get(message.order_validation_text)
if cached_result is None:
message_list = self.getDependentMessageList(activity_tool, message)
transaction.commit() # Release locks.
message_list = activity_tool.getDependentMessageList(message, self)
if message_list:
# The result is not empty, so this message is not executable.
validation_text_dict[message.order_validation_text] = 0
......@@ -189,9 +134,6 @@ class Queue(object):
elif cached_result:
message_dict[message.uid] = message
def hasActivity(self, activity_tool, object, processing_node=None, active_process=None, **kw):
return 0
def flush(self, activity_tool, object, **kw):
pass
......@@ -201,7 +143,7 @@ class Queue(object):
key_list = message.activity_kw.keys()
key_list.sort()
for key in key_list:
method_id = "_validate_%s" % key
method_id = "_validate_" + key
if getattr(self, method_id, None) is not None:
order_validation_item_list.append((key, message.activity_kw[key]))
if len(order_validation_item_list) == 0:
......@@ -216,14 +158,6 @@ class Queue(object):
def getMessageList(self, activity_tool, processing_node=None,**kw):
return []
def countMessage(self, activity_tool,**kw):
return 0
def countMessageWithTag(self, activity_tool,value):
"""Return the number of messages which match the given tag.
"""
return self.countMessage(activity_tool, tag=value)
# Transaction Management
def prepareQueueMessageList(self, activity_tool, message_list):
# Called to prepare transaction commit for queued messages
......
......@@ -26,6 +26,7 @@
#
##############################################################################
from Shared.DC.ZRDB.Results import Results
from Products.CMFActivity.ActivityTool import Message
import sys
#from time import time
......@@ -35,11 +36,6 @@ import transaction
from zLOG import TRACE, WARNING
# Stop validating more messages when this limit is reached
MAX_VALIDATED_LIMIT = 1000
# Read up to this number of messages to validate.
READ_MESSAGE_LIMIT = 1000
class SQLDict(SQLBase):
"""
A simple OOBTree based queue. It should be compatible with transactions
......@@ -74,8 +70,9 @@ class SQLDict(SQLBase):
message_list = activity_buffer.getMessageList(self)
return [m for m in message_list if m.is_registered]
def getProcessableMessageLoader(self, activity_tool, processing_node):
def getProcessableMessageLoader(self, db, processing_node):
path_and_method_id_dict = {}
quote = db.string_literal
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path and
......@@ -87,6 +84,8 @@ class SQLDict(SQLBase):
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
sql_method_id = " AND method_id = %s AND group_method_id = %s" % (
quote(method_id), quote(line.group_method_id))
m = Message.load(line.message, uid=uid, line=line)
merge_parent = m.activity_kw.get('merge_parent')
try:
......@@ -101,11 +100,14 @@ class SQLDict(SQLBase):
path_list.append(path)
uid_list = []
if path_list:
result = activity_tool.SQLDict_selectParentMessage(
path=path_list,
method_id=method_id,
group_method_id=line.group_method_id,
processing_node=processing_node)
# Select parent messages.
result = Results(db.query("SELECT * FROM message"
" WHERE processing_node IN (0, %s) AND path IN (%s)%s"
" ORDER BY path LIMIT 1 FOR UPDATE" % (
processing_node,
','.join(map(quote, path_list)),
sql_method_id,
), 0))
if result: # found a parent
# mark child as duplicate
uid_list.append(uid)
......@@ -115,29 +117,32 @@ class SQLDict(SQLBase):
uid = line.uid
m = Message.load(line.message, uid=uid, line=line)
# return unreserved similar children
result = activity_tool.SQLDict_selectChildMessageList(
path=line.path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = [x.uid for x in result]
path = line.path
result = db.query("SELECT uid FROM message"
" WHERE processing_node = 0 AND (path = %s OR path LIKE %s)"
"%s FOR UPDATE" % (
quote(path), quote(path.replace('_', r'\_') + '/%'),
sql_method_id,
), 0)[1]
reserve_uid_list = [x for x, in result]
uid_list += reserve_uid_list
if not line.processing_node:
# reserve found parent
reserve_uid_list.append(uid)
else:
result = activity_tool.SQLDict_selectDuplicatedLineList(
path=path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = uid_list = [x.uid for x in result]
# Select duplicates.
result = db.query("SELECT uid FROM message"
" WHERE processing_node = 0 AND path = %s%s FOR UPDATE" % (
quote(path), sql_method_id,
), 0)[1]
reserve_uid_list = uid_list = [x for x, in result]
if reserve_uid_list:
activity_tool.SQLDict_reserveDuplicatedLineList(
processing_node=processing_node, uid=reserve_uid_list)
self.assignMessageList(db, processing_node, reserve_uid_list)
else:
activity_tool.SQLDict_commit() # release locks
db.query("COMMIT") # XXX: useful ?
except:
self._log(WARNING, 'getDuplicateMessageUidList got an exception')
activity_tool.SQLDict_rollback() # release locks
self._log(WARNING, 'Failed to reserve duplicates')
db.query("ROLLBACK")
raise
if uid_list:
self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
......
......@@ -31,7 +31,7 @@ from zLOG import LOG, TRACE, INFO, WARNING, ERROR, PANIC
import MySQLdb
from MySQLdb.constants.ER import DUP_ENTRY
from SQLBase import (
SQLBase, sort_message_key, MAX_MESSAGE_LIST_SIZE,
SQLBase, sort_message_key,
UID_SAFE_BITSIZE, UID_ALLOCATION_TRY_COUNT,
)
from Products.CMFActivity.ActivityTool import Message
......@@ -45,77 +45,103 @@ class SQLJoblib(SQLDict):
sql_table = 'message_job'
uid_group = 'portal_activity_job'
def initialize(self, activity_tool, clear):
"""
Initialize the message table using MYISAM Engine
"""
folder = activity_tool.getPortalObject().portal_skins.activity
try:
createMessageTable = folder.SQLJoblib_createMessageTable
except AttributeError:
return
if clear:
folder.SQLBase_dropMessageTable(table=self.sql_table)
createMessageTable()
else:
src = createMessageTable._upgradeSchema(create_if_not_exists=1,
initialize=self._initialize,
table=self.sql_table)
if src:
LOG('CMFActivity', INFO, "%r table upgraded\n%s"
% (self.sql_table, src))
def createTableSQL(self):
return """\
CREATE TABLE %s (
`uid` BIGINT UNSIGNED NOT NULL,
`date` DATETIME(6) NOT NULL,
`path` VARCHAR(255) NOT NULL,
`active_process_uid` INT UNSIGNED NULL,
`method_id` VARCHAR(255) NOT NULL,
`processing_node` SMALLINT NOT NULL DEFAULT -1,
`priority` TINYINT NOT NULL DEFAULT 0,
`group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
`tag` VARCHAR(255) NOT NULL,
`signature` BINARY(16) NOT NULL,
`serialization_tag` VARCHAR(255) NOT NULL,
`retry` TINYINT UNSIGNED NOT NULL DEFAULT 0,
`message` LONGBLOB NOT NULL,
PRIMARY KEY (`uid`),
KEY `processing_node_priority_date` (`processing_node`, `priority`, `date`),
KEY `node_group_priority_date` (`processing_node`, `group_method_id`, `priority`, `date`),
KEY `serialization_tag_processing_node` (`serialization_tag`, `processing_node`),
KEY (`path`),
KEY (`active_process_uid`),
KEY (`method_id`),
KEY (`tag`)
) ENGINE=InnoDB""" % self.sql_table
def generateMessageUID(self, m):
return (tuple(m.object_path), m.method_id, m.activity_kw.get('signature'),
m.activity_kw.get('tag'), m.activity_kw.get('group_id'))
_insert_template = ("INSERT INTO %s (uid,"
" path, active_process_uid, date, method_id, processing_node,"
" priority, group_method_id, tag, signature, serialization_tag,"
" message) VALUES\n(%s)")
def prepareQueueMessageList(self, activity_tool, message_list):
registered_message_list = [m for m in message_list if m.is_registered]
portal = activity_tool.getPortalObject()
for i in xrange(0, len(registered_message_list), MAX_MESSAGE_LIST_SIZE):
message_list = registered_message_list[i:i+MAX_MESSAGE_LIST_SIZE]
path_list = ['/'.join(m.object_path) for m in message_list]
active_process_uid_list = [m.active_process_uid for m in message_list]
method_id_list = [m.method_id for m in message_list]
priority_list = [m.activity_kw.get('priority', 1) for m in message_list]
date_list = [m.activity_kw.get('at_date') for m in message_list]
group_method_id_list = [m.getGroupId() for m in message_list]
tag_list = [m.activity_kw.get('tag', '') for m in message_list]
signature_list=[m.activity_kw.get('signature', '') for m in message_list]
serialization_tag_list = [m.activity_kw.get('serialization_tag', '')
for m in message_list]
processing_node_list = []
for m in message_list:
m.order_validation_text = x = self.getOrderValidationText(m)
processing_node_list.append(0 if x == 'none' else -1)
db = activity_tool.getSQLConnection()
quote = db.string_literal
def insert(reset_uid):
values = self._insert_separator.join(values_list)
del values_list[:]
for _ in xrange(UID_ALLOCATION_TRY_COUNT):
if reset_uid:
reset_uid = False
# Overflow will result into IntegrityError.
db.query("SET @uid := %s" % getrandbits(UID_SAFE_BITSIZE))
try:
portal.SQLJoblib_writeMessage(
uid_list=[
getrandbits(UID_SAFE_BITSIZE)
for _ in xrange(len(message_list))
],
path_list=path_list,
active_process_uid_list=active_process_uid_list,
method_id_list=method_id_list,
priority_list=priority_list,
message_list=map(Message.dump, message_list),
group_method_id_list=group_method_id_list,
date_list=date_list,
tag_list=tag_list,
processing_node_list=processing_node_list,
signature_list=signature_list,
serialization_tag_list=serialization_tag_list)
db.query(self._insert_template % (self.sql_table, values))
except MySQLdb.IntegrityError, (code, _):
if code != DUP_ENTRY:
raise
reset_uid = True
else:
break
else:
raise ValueError("Maximum retry for SQLBase_writeMessageList reached")
raise ValueError("Maximum retry for prepareQueueMessageList reached")
i = 0
reset_uid = True
values_list = []
max_payload = self._insert_max_payload
sep_len = len(self._insert_separator)
for m in message_list:
if m.is_registered:
active_process_uid = m.active_process_uid
order_validation_text = m.order_validation_text = \
self.getOrderValidationText(m)
date = m.activity_kw.get('at_date')
row = ','.join((
'@uid+%s' % i,
quote('/'.join(m.object_path)),
'NULL' if active_process_uid is None else str(active_process_uid),
"UTC_TIMESTAMP(6)" if date is None else quote(render_datetime(date)),
quote(m.method_id),
'0' if order_validation_text == 'none' else '-1',
str(m.activity_kw.get('priority', 1)),
quote(m.getGroupId()),
quote(m.activity_kw.get('tag', '')),
quote(m.activity_kw.get('signature', '')),
quote(m.activity_kw.get('serialization_tag', '')),
quote(Message.dump(m))))
i += 1
n = sep_len + len(row)
max_payload -= n
if max_payload < 0:
if values_list:
insert(reset_uid)
reset_uid = False
max_payload = self._insert_max_payload - n
else:
raise ValueError("max_allowed_packet too small to insert message")
values_list.append(row)
if values_list:
insert(reset_uid)
def getProcessableMessageLoader(self, activity_tool, processing_node):
def getProcessableMessageLoader(self, db, processing_node):
path_and_method_id_dict = {}
quote = db.string_literal
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path, method_id
......@@ -128,19 +154,21 @@ class SQLJoblib(SQLDict):
if original_uid is None:
m = Message.load(line.message, uid=uid, line=line)
try:
result = activity_tool.SQLJoblib_selectDuplicatedLineList(
path=path,
method_id=method_id,
group_method_id=line.group_method_id,
signature=line.signature)
reserve_uid_list = uid_list = [x.uid for x in result]
if reserve_uid_list:
activity_tool.SQLBase_reserveMessageList(
table=self.sql_table,
processing_node=processing_node,
uid=reserve_uid_list)
# Select duplicates.
result = db.query("SELECT uid FROM message_job"
" WHERE processing_node = 0 AND path = %s AND signature = %s"
" AND method_id = %s AND group_method_id = %s FOR UPDATE" % (
quote(path), quote(line.signature),
quote(method_id), quote(line.group_method_id),
), 0)[1]
uid_list = [x for x, in result]
if uid_list:
self.assignMessageList(db, processing_node, uid_list)
else:
db.query("COMMIT") # XXX: useful ?
except:
self._log(WARNING, 'getDuplicateMessageUidList got an exception')
self._log(WARNING, 'Failed to reserve duplicates')
db.query("ROLLBACK")
raise
if uid_list:
self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
......
......@@ -57,6 +57,7 @@ from Products.ERP5Type.UnrestrictedMethod import PrivilegedUser
from zope.site.hooks import setSite
import transaction
from App.config import getConfiguration
from Shared.DC.ZRDB.Results import Results
import Products.Localizer.patches
localizer_lock = Products.Localizer.patches._requests_lock
......@@ -191,7 +192,6 @@ class Message(BaseMessage):
call_traceback = None
exc_info = None
is_executed = MESSAGE_NOT_EXECUTED
processing = None
traceback = None
oid = None
is_registered = False
......@@ -367,11 +367,6 @@ class Message(BaseMessage):
except:
self.setExecutionState(MESSAGE_NOT_EXECUTED, context=activity_tool)
def validate(self, activity, activity_tool, check_order_validation=1):
return activity.validate(activity_tool, self,
check_order_validation=check_order_validation,
**self.activity_kw)
def notifyUser(self, activity_tool, retry=False):
"""Notify the user that the activity failed."""
portal = activity_tool.getPortalObject()
......@@ -655,11 +650,6 @@ class ActivityTool (BaseTool):
activity_timing_log = False
cancel_and_invoke_links_hidden = False
def SQLDict_setPriority(self, **kw):
real_SQLDict_setPriority = getattr(self.aq_parent, 'SQLDict_setPriority')
LOG('ActivityTool', 0, real_SQLDict_setPriority(src__=1, **kw))
return real_SQLDict_setPriority(**kw)
# Filter content (ZMI))
def filtered_meta_types(self, user=None):
# Filters the list of available meta types.
......@@ -670,6 +660,9 @@ class ActivityTool (BaseTool):
meta_types.append(meta_type)
return meta_types
def getSQLConnection(self):
return self.aq_inner.aq_parent.cmf_activity_sql_connection()
def maybeMigrateConnectionClass(self):
connection_id = 'cmf_activity_sql_connection'
sql_connection = getattr(self, connection_id, None)
......@@ -689,6 +682,20 @@ class ActivityTool (BaseTool):
self.maybeMigrateConnectionClass()
for activity in activity_dict.itervalues():
activity.initialize(self, clear=False)
# Remove old skin if any.
skins_tool = self.getPortalObject().portal_skins
name = 'activity'
if (getattr(skins_tool.get(name), '_dirpath', None)
== 'Products.CMFActivity:skins/activity'):
for selection, skins in skins_tool.getSkinPaths():
skins = skins.split(',')
try:
skins.remove(name)
except ValueError:
continue
skins_tool.manage_skinLayers(
add_skin=1, skinname=selection, skinpath=skins)
skins_tool._delObject(name)
def _callSafeFunction(self, batch_function):
return batch_function()
......@@ -1127,14 +1134,16 @@ class ActivityTool (BaseTool):
def hasActivity(self, *args, **kw):
# Check in each queue if the object has deferred tasks
# if not argument is provided, then check on self
if len(args) > 0:
obj = args[0]
if args:
obj, = args
else:
obj = self
for activity in activity_dict.itervalues():
if activity.hasActivity(aq_inner(self), obj, **kw):
return True
return False
path = None if obj is None else '/'.join(obj.getPhysicalPath())
db = self.getSQLConnection()
quote = db.string_literal
return bool(db.query("(%s)" % ") UNION ALL (".join(
activity.hasActivitySQL(quote, path=path, **kw)
for activity in activity_dict.itervalues()))[1])
security.declarePrivate('getActivityBuffer')
def getActivityBuffer(self, create_if_not_found=True):
......@@ -1443,8 +1452,9 @@ class ActivityTool (BaseTool):
"""
if not(isinstance(message_uid_list, list)):
message_uid_list = [message_uid_list]
self.SQLBase_makeMessageListAvailable(table=activity_dict[activity].sql_table,
uid=message_uid_list)
if message_uid_list:
activity_dict[activity].unreserveMessageList(self.getSQLConnection(),
0, message_uid_list)
if REQUEST is not None:
return REQUEST.RESPONSE.redirect('%s/%s' % (
self.absolute_url(), 'view'))
......@@ -1470,8 +1480,8 @@ class ActivityTool (BaseTool):
"""
if not(isinstance(message_uid_list, list)):
message_uid_list = [message_uid_list]
self.SQLBase_delMessage(table=activity_dict[activity].sql_table,
uid=message_uid_list)
activity_dict[activity].deleteMessageList(
self.getSQLConnection(), message_uid_list)
if REQUEST is not None:
return REQUEST.RESPONSE.redirect('%s/%s' % (
self.absolute_url(), 'view'))
......@@ -1523,10 +1533,7 @@ class ActivityTool (BaseTool):
"""
Return the number of messages which match the given tag.
"""
message_count = 0
for activity in activity_dict.itervalues():
message_count += activity.countMessageWithTag(aq_inner(self), value)
return message_count
return self.countMessage(tag=value)
security.declarePublic('countMessage')
def countMessage(self, **kw):
......@@ -1540,10 +1547,11 @@ class ActivityTool (BaseTool):
tag : activities with a particular tag
message_uid : activities with a particular uid
"""
message_count = 0
for activity in activity_dict.itervalues():
message_count += activity.countMessage(aq_inner(self), **kw)
return message_count
db = self.getSQLConnection()
quote = db.string_literal
return sum(x for x, in db.query("(%s)" % ") UNION ALL (".join(
activity.countMessageSQL(quote, **kw)
for activity in activity_dict.itervalues()))[1])
security.declareProtected( CMFCorePermissions.ManagePortal , 'newActiveProcess' )
def newActiveProcess(self, REQUEST=None, **kw):
......@@ -1554,23 +1562,31 @@ class ActivityTool (BaseTool):
REQUEST['RESPONSE'].redirect( 'manage_main' )
return obj
# Active synchronisation methods
security.declarePrivate('validateOrder')
def validateOrder(self, message, validator_id, validation_value):
message_list = self.getDependentMessageList(message, validator_id, validation_value)
return len(message_list) > 0
security.declarePrivate('getDependentMessageList')
def getDependentMessageList(self, message, validator_id, validation_value):
message_list = []
method_id = "_validate_" + validator_id
def getDependentMessageList(self, message, validating_queue=None):
activity_kw = message.activity_kw
db = self.getSQLConnection()
quote = db.string_literal
queries = []
for activity in activity_dict.itervalues():
method = getattr(activity, method_id, None)
if method is not None:
result = method(aq_inner(self), message, validation_value)
if result:
message_list += [(activity, m) for m in result]
return message_list
q = activity.getValidationSQL(
quote, activity_kw, activity is validating_queue)
if q:
queries.append(q)
if queries:
message_list = []
for line in Results(db.query("(%s)" % ") UNION ALL (".join(queries))):
activity = activity_dict[line.activity]
m = Message.load(line.message,
line=line,
uid=line.uid,
date=line.date,
processing_node=line.processing_node)
if not hasattr(m, 'order_validation_text'): # BBB
m.order_validation_text = activity.getOrderValidationText(m)
message_list.append((activity, m))
return message_list
return ()
# Required for tests (time shift)
def timeShift(self, delay):
......
#!/bin/sh
set -e
# Small watching script based on Sébastien idea.
# ideas:
# - more control on what would be displayed
......@@ -32,13 +31,47 @@ INTERVAL=$2
exit 1
}
SELECT=""
for t in message message_queue ; do
SELECT=$SELECT"""
SELECT count(*) AS $t, ${text_group:-method_id}, processing, processing_node AS node, min(priority) AS min_pri, max(priority) AS max_pri FROM $t GROUP BY ${text_group:-method_id}, processing, processing_node ORDER BY node;
SELECT count(*) AS $t, processing, processing_node, min(priority) AS min_pri, max(priority) AS max_pri FROM $t GROUP BY processing, processing_node;
SELECT priority as pri, MIN(timediff(NOW(), date)) AS min, AVG(timediff(NOW() , date)) AS avg, MAX(timediff(NOW() , date)) AS max FROM $t GROUP BY priority;
SELECT count(*) AS ${t}_count FROM $t;
"""
node_priority_cols="processing_node AS node, MIN(priority) AS min_pri, MAX(priority) AS max_pri"
for t in message:dict message_queue:queue message_job:joblib; do
table=${t%:*}
t=${t#*:}
create=$create"
CREATE TEMPORARY TABLE _$t(
n INT UNSIGNED NOT NULL,
${text_group:-method_id} VARCHAR(255) NOT NULL,
processing_node SMALLINT NOT NULL,
priority TINYINT NOT NULL,
min_date DATETIME(6) NOT NULL,
max_date DATETIME(6) NOT NULL,
max_retry TINYINT UNSIGNED NOT NULL
) ENGINE=MEMORY;"
collect=$collect"
INSERT INTO _$t SELECT count(*), ${text_group:-method_id},
processing_node, priority, MIN(date), MAX(date), MAX(retry) FROM $table
GROUP BY processing_node, priority, ${text_group:-method_id};"
select=$select"
SELECT IFNULL(SUM(n),0) AS $t, ${text_group:-method_id},
$node_priority_cols, MAX(max_retry) AS max_retry FROM _$t
GROUP BY processing_node, ${text_group:-method_id}
ORDER BY processing_node, ${text_group:-method_id};
SELECT priority,
TIME_FORMAT(TIMEDIFF(UTC_TIMESTAMP(6), MAX(max_date)), \"%T\") AS min,
TIME_FORMAT(TIMEDIFF(UTC_TIMESTAMP(6), MIN(min_date)), \"%T\") AS max
FROM _$t GROUP BY priority ORDER BY priority;"