py2/py3: Fix __import__ fromlist argument.

__import__ `fromlist` argument was wrong. It was working anyway with Python2 but
not anymore with Python3, raising a `ModuleNotFoundError` exception. According
to Python `__import__(name, globals, locals, fromlist)` documentation:

  When the `name` variable is of the form `package.module`, normally, the
  top-level package (the `name` up till the first dot) is returned, *not* the
  module named by `name`. However, when a non-empty `fromlist` argument is
  given, the module named by `name` is returned.

Thus, the following patterns were wrong:
  * __import__(MODULE_NAME, globals(), locals(), MODULE_NAME)
    => Iterate through each character of MODULE_NAME as fromlist is expected to
       be a list/tuple.
  * __import__(MODULE_NAME, globals(), locals(), [MODULE_NAME])
    => This works but actually tries to import MODULE_NAME object from
       MODULE_NAME module (no error if it cannot).

The goal of such __import__ calls were for __import__ to return the right-end
module instead of the top-level package. In such case, `fromlist=['']` is the
way to go as it __import__ does not check if the object exists in the module if
it's an empty string. However, it is even better and easier to read to use
importlib.import_module() for that...

Also, add `from __future__ import absolute_import` because python2 tries both
relative and absolute import (level=-1 __import__ parameter) whereas python3
does absolute import by default (level=0).
Co-authored-by: Kazuhiko SHIOZAKI <kazuhiko@nexedi.com>

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5TypeInterfaces.py

@@ -58,18 +58,16 @@ class TestERP5TypeInterfaces(unittest.TestCase):
 def makeTestMethod(import_tuple, interface):
   """Common method which checks if documents implements interface"""
   def testMethod(self):
-    Klass = getattr(
-      __import__(import_tuple[0], globals(), locals(), [import_tuple[0]]),
-      import_tuple[1])
+    from importlib import import_module
+
+    Klass = getattr(import_module(import_tuple[0]), import_tuple[1])
 
     import Products.ERP5Type.interfaces
     try:
       Interface = getattr(Products.ERP5Type.interfaces, interface)
     except AttributeError:
       InterfaceModuleName = 'erp5.component.interface.%s' % interface
-      Interface = getattr(
-        __import__(InterfaceModuleName, globals(), locals(), [InterfaceModuleName]),
-        interface)
+      Interface = getattr(import_module(InterfaceModuleName), interface)
 
     verifyClass(Interface, Klass)
 

bt5/erp5_syncml/ModuleComponentTemplateItem/portal_components/module.erp5.XMLSyncUtils.py

@@ -128,18 +128,19 @@ def getConduitByName(conduit_name):
   Conduit can also be defined as Extension to have it editable through the web, in this
   case its definition must be Extensions.<Conduit Module>
   """
+  from importlib import import_module
   if conduit_name.startswith('Products'):
     path = conduit_name
     conduit_name = conduit_name.split('.')[-1]
-    conduit_module = __import__(path, globals(), locals(), [''])
+    conduit_module = import_module(path)
   elif conduit_name.startswith('Extensions'):
-    conduit_module = __import__(conduit_name, globals(), locals(), [''])
+    conduit_module = import_module(conduit_name)
     conduit_name = conduit_name.split('.')[-1]
   elif conduit_name.startswith('extension.'):
-    conduit_module = __import__("erp5.component."+conduit_name, globals(), locals(), [''])
+    conduit_module = import_module("erp5.component." + conduit_name)
     conduit_name = conduit_name.split('.')[-1]
   else:
-    conduit_module = __import__('erp5.component.module.'+conduit_name, globals(), locals(), [''])
+    conduit_module = import_module('erp5.component.module.' + conduit_name)
   conduit_instance = getattr(conduit_module, conduit_name)()
   return conduit_instance
 

bt5/erp5_web_service/ToolComponentTemplateItem/portal_components/tool.erp5.WebServiceTool.py

@@ -55,15 +55,14 @@ handler_module_dict = {
   'sql' : "SQLConnection",
   'document' : "DocumentConnection",
 }
+from importlib import import_module
 for handler_id, module_id in handler_module_dict.iteritems():
   # Ignore non-functionnal plugins.
   # This is done to avoid adding strict dependencies.
   # Code relying on the presence of a plugin will fail upon
   # WebServiceTool.connect .
   try:
-    module = __import__(
-      'erp5.component.module.%s' % (module_id, ),
-      globals(), {}, [module_id])
+    module = import_module('erp5.component.module.' + module_id)
   except ImportError:
     LOG('WebServiceTool', WARNING,
         'Unable to import module %r. %r transport will not be available.' % \

product/ERP5/Document/BusinessTemplate.py

@@ -90,6 +90,7 @@ from xml.sax.saxutils import escape
 from Products.CMFCore.Expression import Expression
 from six.moves.urllib.parse import quote, unquote, urlparse
 from difflib import unified_diff
+from importlib import import_module
 import posixpath
 import transaction
 import inspect
@@ -6858,9 +6859,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
             if component_portal_type in ('Document Component',
                                          'Tool Component'):
               try:
-                klass = getattr(
-                  __import__(source_reference, {}, {}, [source_reference]),
-                  subsubmodule_name)
+                klass = getattr(import_module(source_reference), subsubmodule_name)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),
@@ -6890,7 +6889,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
               # Generally: foo_bar.py => IFooBar, but to avoid quirks (such as
               # 'sql_foo.py' => 'ISQLFoo'), get the Interface class __name__
               try:
-                interface_module = __import__(source_reference, {}, {}, source_reference)
+                interface_module = import_module(source_reference)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),
@@ -6919,7 +6918,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
             # TODO-arnau: Refactor with 'Interface Component'
             elif component_portal_type == 'Mixin Component':
               try:
-                mixin_module = __import__(source_reference, {}, {}, source_reference)
+                mixin_module = import_module(source_reference)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),

product/ERP5Type/Utils.py

@@ -1076,7 +1076,8 @@ def importLocalDocument(class_id, path=None, class_path=None):
   if class_path:
     assert path is None
     module_path = class_path.rsplit('.', 1)[0]
-    module = __import__(module_path, {}, {}, (module_path,))
+    from importlib import import_module
+    module = import_module(module_path)
     try:
       klass = getattr(module, class_id)
     except AttributeError:

product/ERP5Type/dynamic/portal_type_class.py

@@ -63,9 +63,10 @@ ACQUIRE_LOCAL_ROLE_GETTER_DICT = {
 
 
 def _importFilesystemClass(classpath):
+  from importlib import import_module
   try:
     module_path, class_name = classpath.rsplit('.', 1)
-    module = __import__(module_path, {}, {}, (module_path,))
+    module = import_module(module_path)
     klass = getattr(module, class_name)
 
     # XXX is this required? (here?)

product/ERP5Type/mixin/component.py

@@ -380,9 +380,9 @@ class ComponentMixin(with_metaclass(RecordablePropertyMetaClass, type('NewBase',
     if source_reference is None or not source_reference.startswith('Products'):
       path = os.path.join(cls._getFilesystemPath(), reference + '.py')
     else:
+      from importlib import import_module
+      module_obj = import_module(source_reference)
       import inspect
-      module_obj = __import__(source_reference, globals(), {},
-                              level=0, fromlist=[source_reference])
       path = inspect.getsourcefile(module_obj)
 
     with open(path) as f:

product/ERP5Type/patches/DA.py

@@ -277,11 +277,9 @@ def getObjectMeta(original_function):
   def getObject(module, name, reload=0):
     # Modified version that ignore errors as long as the module can be be
     # imported, which is enough to use a ZODB Extension as a brain.
+    from importlib import import_module
     try:
-      m = __import__('erp5.component.extension.%s' % module, globals(),
-                     {}, 'erp5.component.extension')
-
-      o = getattr(m, name, None)
+      o = getattr(import_module('erp5.component.extension.%s' % module), name, None)
       if o is None:
         raise ImportError(
           "Cannot get %s from erp5.component.extension.%s" % (name, module))

product/ERP5Type/patches/pylint.py

@@ -23,6 +23,7 @@ import six
 import sys
 import types
 import warnings
+import importlib
 from Products.ERP5Type import IS_ZOPE2
 
 # TODO: make sure that trying to use it does not import isort, because the
@@ -245,10 +246,7 @@ def _getattr(self, name, *args, **kw):
                 # XXX actually maybe we don't need this branch at all on py3
             ):
             raise
-        real_module = __import__(
-            self.name,
-            fromlist=[self.name] if six.PY2 else [name],
-            level=0)
+        real_module = importlib.import_module(self.name)
         try:
             attr = getattr(real_module, name)
         except AttributeError:
@@ -462,7 +460,7 @@ def fail_hook_BTrees(modname):
     if modname not in _inspected_modules:
         try:
             modcode = build_stub(
-              __import__(modname, {}, {}, [modname], level=0),
+              importlib.import_module(modname),
               # Exclude all classes ending with 'Py' (no reason to not call the
               # C version and not part of public API anyway)
               identifier_re=r'^[A-Za-z_]\w*(?<!Py)$')
@@ -507,7 +505,7 @@ for filename in os.listdir(os.path.dirname(lxml.__file__)):
         module_name = 'lxml.' + filename.split('.', 1)[0]
         _register_module_extender_from_live_module(
             module_name,
-            __import__(module_name, fromlist=[module_name], level=0))
+            importlib.import_module(module_name))
 
 # Wendelin and XLTE are special namespace packages which pylint fails to recognize, and so
 # complains about things like `from wendelin.bigarray.array_zodb import ZBigArray`
@@ -528,7 +526,6 @@ def register_xpkg(pkgname):
                 return m
             MANAGER.register_transform(Module, xpkg_transform, lambda node: node.name == pkgname)
         else:
-            import importlib
             def fail_hook_xpkg(modname):
                 if modname.split('.')[0] == pkgname:
                     return MANAGER.ast_from_module(importlib.import_module(modname))

product/ERP5Type/tests/runUnitTest.py

 #!/usr/bin/env python2.7
+from __future__ import absolute_import
 from __future__ import print_function
 import os
 import sys
@@ -298,10 +299,9 @@ class ERP5TypeTestLoader(unittest.TestLoader):
         self._loading_packages = set()
 
   def _importZodbTestComponent(self, name):
+    from importlib import import_module
     import erp5.component.test
-    module = __import__('erp5.component.test.' + name,
-                        fromlist=['erp5.component.test'],
-                        level=0)
+    module = import_module('erp5.component.test.' + name)
     try:
       self._test_component_ref_list.append(module)
     except AttributeError:

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -28,6 +28,7 @@
 # 02110-1301, USA.
 #
 ##############################################################################
+from __future__ import absolute_import
 
 import gc
 import os
@@ -37,6 +38,7 @@ import unittest
 import warnings
 import re
 import sys
+from importlib import import_module
 
 import transaction
 from persistent import Persistent
@@ -1449,8 +1451,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
   def afterSetUp(self):
     self._component_tool = self.portal.portal_components
-    self._module = __import__(self._document_class._getDynamicModuleNamespace(),
-                              fromlist=['erp5.component'])
+    self._module = import_module(self._document_class._getDynamicModuleNamespace())
     self._component_tool.reset(force=True,
                                reset_portal_type_at_transaction_boundary=True)
 
@@ -1520,7 +1521,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
     if expected_default_version is not None:
       top_module_name = self._document_class._getDynamicModuleNamespace()
-      top_module = __import__(top_module_name, level=0, fromlist=[top_module_name])
+      top_module = import_module(top_module_name)
 
       # The module must be available in its default version
       self.assertHasAttribute(top_module, expected_default_version)
@@ -1549,10 +1550,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
   def _importModule(self, module_name):
     module_name = self._getComponentFullModuleName(module_name)
-    module = __import__(
-      module_name,
-      fromlist=[self._document_class._getDynamicModuleNamespace()],
-      level=0)
+    module = import_module(module_name)
     self.assertIn(module_name, sys.modules)
     return module
 
@@ -2043,8 +2041,7 @@ def bar(*args, **kwargs):
     # later that the module has not been added to the top-level package
     self.assertModuleImportable('erp5_version.%s' % imported_reference)
 
-    top_module = __import__(top_module_name, level=0,
-                            fromlist=[top_module_name])
+    top_module = import_module(top_module_name)
 
     self._importModule('erp5_version.%s' % imported_reference)
 
@@ -2106,8 +2103,7 @@ def function_foo(*args, **kwargs):
       self.failIfModuleImportable('foo_version.%s' % reference)
 
       top_module_name = self._document_class._getDynamicModuleNamespace()
-      top_module = __import__(top_module_name, level=0,
-                              fromlist=[top_module_name])
+      top_module = import_module(top_module_name)
 
       self._importModule(reference)
       module = getattr(top_module, reference)

product/PortalTransforms/Transform.py

 # -*- coding: utf-8 -*-
 from zLOG import ERROR
 from six.moves import UserDict
+from importlib import import_module
 
 from zope.interface import implementer
 
@@ -21,7 +22,7 @@ from Products.PortalTransforms.transforms.broken import BrokenTransform
 
 def import_from_name(module_name):
     """ import and return a module by its name """
-    return __import__(module_name, {}, {}, module_name)
+    return import_module(module_name)
 
 def make_config_persistent(kwargs):
     """ iterates on the given dictionnary and replace list by persistent list,