Commit 0b3aecff authored by Julien Muchembled's avatar Julien Muchembled

CMFActivity: better date ordering by using micro-precision

Originally, uids somehow sorted messages by date of insertion, in particular
for those that were created within the same second. But since random uids,
such messages became validated or processed in random order.

Note however that by default, messages created in the same transaction all have
exactly the same date, so commit a42da4de
("CMFActivity: Do not use offset for scanning messages to validate.")
forces us to keep the ordering on uids (in addition to priority/date).

Existing instances will upgrade automatically, using the already existing code
to upgrade tables in a generic way. You should see the following logs:

  INFO CMFActivity 'message_queue' table upgraded
  ALTER TABLE message_queue
    MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
    MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing
  INFO CMFActivity 'message_job' table upgraded
  ALTER TABLE message_job
    MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
    MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing
  INFO CMFActivity 'message' table upgraded
  ALTER TABLE message
    MODIFY COLUMN date datetime(6) NOT NULL AFTER uid,
    MODIFY COLUMN processing_date datetime(6) DEFAULT NULL AFTER processing
parent e32deab9
......@@ -75,6 +75,8 @@ _DequeueMessageException = Exception()
def sqltest_dict():
sqltest_dict = {}
no_quote_type = int, float, long
def render_datetime(x):
return "%.4d-%.2d-%.2d %.2d:%.2d:%09.6f" % x.toZone('UTC').parts()[:6]
def _(name, column=None, op="="):
if column is None:
column = name
......@@ -83,18 +85,17 @@ def sqltest_dict():
if isinstance(value, no_quote_type):
return column_op + str(value)
if isinstance(value, DateTime):
value = value.toZone('UTC').ISO()
value = render_datetime(value)
if isinstance(value, basestring):
return column_op + render_string(value)
assert op == "=", value
if value is None: # XXX: see comment in SQLBase._getMessageList
return column + " IS NULL"
for x in value:
if isinstance(x, no_quote_type):
render_string = str
elif isinstance(x, DateTime):
value = (x.toZone('UTC').ISO() for x in value)
return "%s IN (%s)" % (column, ', '.join(map(render_string, value)))
return "%s IN (%s)" % (column, ', '.join(map(
str if isinstance(x, no_quote_type) else
render_datetime if isinstance(x, DateTime) else
render_string, value)))
return "0"
sqltest_dict[name] = render
_('active_process_uid')
......
......@@ -10,13 +10,13 @@ class_file:
<params>table</params>
CREATE TABLE <dtml-var table> (
`uid` BIGINT UNSIGNED NOT NULL,
`date` DATETIME NOT NULL,
`date` DATETIME(6) NOT NULL,
`path` VARCHAR(255) NOT NULL,
`active_process_uid` INT UNSIGNED NULL,
`method_id` VARCHAR(255) NOT NULL,
`processing_node` SMALLINT NOT NULL DEFAULT -1,
`processing` TINYINT NOT NULL DEFAULT 0,
`processing_date` DATETIME,
`processing_date` DATETIME(6),
`priority` TINYINT NOT NULL DEFAULT 0,
`group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
`tag` VARCHAR(255) NOT NULL,
......
......@@ -8,4 +8,4 @@ class_name:
class_file:
</dtml-comment>
<params></params>
SELECT UTC_TIMESTAMP()
SELECT UTC_TIMESTAMP(6)
......@@ -13,6 +13,6 @@ SELECT `priority`, `date` FROM
<dtml-var table>
WHERE
processing_node = 0
AND date <= UTC_TIMESTAMP()
AND date <= UTC_TIMESTAMP(6)
ORDER BY priority, date
LIMIT 1
......@@ -19,7 +19,7 @@ FROM
<dtml-var table>
WHERE
processing_node=0
AND date <= <dtml-sqlvar to_date type="datetime">
AND date <= <dtml-sqlvar to_date type="datetime(6)">
<dtml-if expr="group_method_id is not None">
AND group_method_id = <dtml-sqlvar group_method_id type="string">
</dtml-if>
......
......@@ -12,7 +12,7 @@ uid</params>
UPDATE
<dtml-var table>
SET
processing_date = UTC_TIMESTAMP(),
processing_date = UTC_TIMESTAMP(6),
processing = 1
WHERE
<dtml-sqltest uid type="int" multiple>
......
......@@ -15,7 +15,7 @@ delay
UPDATE
<dtml-var table>
SET
date = DATE_ADD(UTC_TIMESTAMP(), INTERVAL
date = DATE_ADD(UTC_TIMESTAMP(6), INTERVAL
<dtml-sqlvar delay type="int"> SECOND)
<dtml-if expr="retry is not None">
, priority = priority + <dtml-sqlvar retry type="int">
......
......@@ -29,7 +29,7 @@ VALUES
<dtml-sqlvar expr="uid_list[loop_item]" type="int">,
<dtml-sqlvar expr="path_list[loop_item]" type="string">,
<dtml-sqlvar expr="active_process_uid_list[loop_item]" type="int" optional>,
<dtml-if expr="date_list is not None"><dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime"><dtml-else>UTC_TIMESTAMP()</dtml-if><dtml-else>UTC_TIMESTAMP()</dtml-if>,
<dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime(6)"><dtml-else>UTC_TIMESTAMP(6)</dtml-if>,
<dtml-sqlvar expr="method_id_list[loop_item]" type="string">,
<dtml-sqlvar expr="processing_node_list[loop_item]" type="int">,
0,
......
......@@ -10,13 +10,13 @@ class_file:
<params></params>
CREATE TABLE message_job (
`uid` BIGINT UNSIGNED NOT NULL,
`date` DATETIME NOT NULL,
`date` DATETIME(6) NOT NULL,
`path` VARCHAR(255) NOT NULL,
`active_process_uid` INT UNSIGNED NULL,
`method_id` VARCHAR(255) NOT NULL,
`processing_node` SMALLINT NOT NULL DEFAULT -1,
`processing` TINYINT NOT NULL DEFAULT 0,
`processing_date` DATETIME,
`processing_date` DATETIME(6),
`priority` TINYINT NOT NULL DEFAULT 0,
`group_method_id` VARCHAR(255) NOT NULL DEFAULT '',
`tag` VARCHAR(255) NOT NULL,
......
......@@ -30,7 +30,7 @@ VALUES
<dtml-sqlvar expr="uid_list[loop_item]" type="int">,
<dtml-sqlvar expr="path_list[loop_item]" type="string">,
<dtml-sqlvar expr="active_process_uid_list[loop_item]" type="int" optional>,
<dtml-if expr="date_list is not None"><dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime"><dtml-else>UTC_TIMESTAMP()</dtml-if><dtml-else>UTC_TIMESTAMP()</dtml-if>,
<dtml-if expr="date_list[loop_item] is not None"><dtml-sqlvar expr="date_list[loop_item]" type="datetime(6)"><dtml-else>UTC_TIMESTAMP(6)</dtml-if>,
<dtml-sqlvar expr="method_id_list[loop_item]" type="string">,
<dtml-sqlvar expr="processing_node_list[loop_item]" type="int">,
0,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment