IdTool: expect bytes for group_id

group_id is used as key of OOBtree and as [documented], it's not
possible to mix keys that can not be compared, so we can not have a mix
of string and bytes.

🤔 a BTree can not contain str and bytes as keys (the same way that on
python2 it can not contain unicode and str), basically we just need a
consistent type and something compatible with how we transform the data
from python2 databases.

bt5/erp5_accounting/TestTemplateItem/portal_components/test.erp5.testAccounting.py

@@ -3413,13 +3413,13 @@ class TestTransactions(AccountingTestCase):
     # ...except uid generator
     new_uid, = portal_ids.generateNewIdList(
       id_generator='uid',
-      id_group='catalog_uid',
+      id_group=b'catalog_uid',
       id_count=1,
     )
     portal_ids.clearGenerator(all=True)
     portal_ids.generateNewIdList(
       id_generator='uid',
-      id_group='catalog_uid',
+      id_group=b'catalog_uid',
       id_count=1,
       default=new_uid,
     )

product/ERP5/Document/IdGenerator.py

@@ -91,9 +91,10 @@ class IdGenerator(Base):
       by BTrees.Length to manage conflict in the zodb, use also a persistant
       mapping to be persistent
     """
-    # For compatibilty with sql data, must not use id_group as a list
-    if not isinstance(id_group, str):
-      raise TypeError('id_group is not a string')
+    # Type of id groups must be consistent, because we use them as BTree keys
+    # https://btrees.readthedocs.io/en/latest/overview.html#total-ordering-and-persistence
+    if not isinstance(id_group, bytes):
+      raise TypeError('id_group must be bytes')
     return self._getLatestSpecialiseValue().generateNewIdList(id_group=id_group,
                                                               id_count=id_count,
                                                               default=default,

product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py

@@ -71,8 +71,10 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
       mapping to be persistent
     """
     # Check the arguments
-    if id_group in (None, 'None'):
+    if id_group in (None, b'None'):
       raise ValueError('%r is not a valid group Id.' % id_group)
+    if not isinstance(id_group, bytes):
+      raise TypeError('id_group must be bytes')
     if default is None:
       default = 0
 
@@ -134,6 +136,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     # the last id stored in the sql table
     for line in self._getValueListFromTable():
       id_group = line['id_group']
+      assert isinstance(id_group, bytes)
       last_id = line['last_id']
       if id_group in self.last_max_id_dict and \
         self.last_max_id_dict[id_group].value > last_id:
@@ -197,6 +200,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
             getattr(portal_ids, 'dict_length_ids', None) is None):
       dump_dict = portal_ids.dict_length_ids
       for id_group, last_id in dump_dict.items():
+        assert isinstance(id_group, bytes)
         last_insert_id = get_last_id_method(id_group=id_group)
         last_id = int(last_id.value)
         if len(last_insert_id) != 0:

product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py

@@ -57,8 +57,10 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
      Use int to store the last_id, use also a persistant mapping for to be
      persistent.
     """
-    if id_group in (None, 'None'):
+    if id_group in (None, b'None'):
       raise ValueError('%r is not a valid group Id.' % id_group)
+    if not isinstance(id_group, bytes):
+      raise TypeError('id_group must be bytes')
     if default is None:
       default = 0
     last_id_dict = getattr(self, 'last_id_dict', None)
@@ -107,8 +109,8 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
     # Dump the dict_ids dictionary
     if getattr(portal_ids, 'dict_ids', None) is not None:
       for id_group, last_id in portal_ids.dict_ids.items():
-        if not isinstance(id_group, str):
-          id_group = repr(id_group)
+        if not isinstance(id_group, bytes):
+          id_group = repr(id_group).encode()
         if id_group in self.last_id_dict and \
            self.last_id_dict[id_group] > last_id:
           continue
@@ -148,7 +150,9 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
       self.clearGenerator()
     if not isinstance(id_dict, dict):
       raise TypeError('the argument given is not a dictionary')
-    for value in id_dict.values():
+    for key, value in id_dict.items():
+      if not isinstance(key, bytes):
+        raise TypeError('key %r given in dictionary is not bytes' % (key, ))
       if not isinstance(value, six.integer_types):
         raise TypeError('the value given in dictionary is not a integer')
     self.last_id_dict.update(id_dict)

product/ERP5/Tool/IdTool.py

@@ -115,12 +115,13 @@ class IdTool(BaseTool):
     """
       Generate the next id in the sequence of ids of a particular group
     """
-    if id_group in (None, 'None'):
+    if id_group in (None, b'None'):
       raise ValueError('%r is not a valid id_group' % id_group)
     # for compatibilty with sql data, must not use id_group as a list
-    if not isinstance(id_group, str):
-      id_group = repr(id_group)
-      warnings.warn('id_group must be a string, other types '
+    if not isinstance(id_group, bytes):
+      # TODO: check that this repr same as python2 !
+      id_group = repr(id_group).encode()
+      warnings.warn('id_group must be bytes, other types '
                     'are deprecated.', DeprecationWarning)
     if id_generator is None:
       id_generator = 'document'
@@ -175,11 +176,11 @@ class IdTool(BaseTool):
     """
       Generate a list of next ids in the sequence of ids of a particular group
     """
-    if id_group in (None, 'None'):
+    if id_group in (None, b'None'):
       raise ValueError('%r is not a valid id_group' % id_group)
     # for compatibilty with sql data, must not use id_group as a list
-    if not isinstance(id_group, str):
-      id_group = repr(id_group)
+    if not isinstance(id_group, bytes):
+      id_group = repr(id_group).encode()
       warnings.warn('id_group must be a string, other types '
                     'are deprecated.', DeprecationWarning)
     if id_generator is None:

product/ERP5Type/Base.py

@@ -28,7 +28,6 @@
 ##############################################################################
 
 from __future__ import absolute_import
-from six import string_types as basestring
 from struct import unpack
 from copy import copy
 import warnings
@@ -3591,7 +3590,7 @@ class Base(
     sequence and for <self> instance, and is monotonously increasing by 1 for
     each generated id.
 
-    group (string):
+    group (bytes):
       Identifies the sequence to use.
     count (int):
       How many identifiers to generate.
@@ -3623,8 +3622,8 @@ class Base(
       It is expected that group creation is a rare event, very unlikely to
       happen concurrently in multiple transactions on the same object.
     """
-    if not isinstance(group, basestring):
-      raise TypeError('group must be a string')
+    if not isinstance(group, bytes):
+      raise TypeError('group must be bytes')
     if not isinstance(default, six.integer_types):
       raise TypeError('default must be an integer')
     if not isinstance(count, six.integer_types):
@@ -3649,7 +3648,7 @@ class Base(
       next_id = default
     new_next_id = None if poison else next_id + count
     id_generator_state[group].value = new_next_id
-    return range(next_id, new_next_id)
+    return ensure_list(range(next_id, new_next_id))
 
 InitializeClass(Base)
 

product/ZSQLCatalog/SQLCatalog.py

@@ -1148,7 +1148,7 @@ class Catalog(Folder,
         uid_buffer.extend(
           self.getPortalObject().portal_ids.generateNewIdList(
             id_generator='uid',
-            id_group='catalog_uid',
+            id_group=b'catalog_uid',
             id_count=UID_BUFFER_SIZE,
             default=getattr(self, '_max_uid', lambda: 1)(),
           ),