jedi wip

bt5/erp5_monaco_editor/ExtensionTemplateItem/portal_components/extension.erp5.Jedi.py

 from __future__ import unicode_literals
 import json
 import sys
-from threading import RLock
+import typing
 import logging
+from threading import RLock
 
 logger = logging.getLogger("erp5.extension.Jedi")
 
@@ -16,45 +17,40 @@ last_reload_time = time.time()
 # XXX I'm really not sure this is needed, jedi seems fast enough.
 jedi.settings.call_signatures_validity = 30
 
-# monkey patch to disable buggy sys.path addition based on buildout.
-# rdiff-backup seem to trigger a bug, but it's generally super slow and not correct for us.
-try:
+if True:
+  # monkey patch to disable buggy sys.path addition based on buildout.
+  # https://github.com/davidhalter/jedi/issues/1325
+  # rdiff-backup seem to trigger a bug, but it's generally super slow and not correct for us.
+  try:
     # in jedi 0.15.1 it's here
     from jedi.evaluate import sys_path as jedi_inference_sys_path  # pylint: disable=import-error,unused-import,no-name-in-module
-except ImportError:
-  # but it's beeing moved. Next release will be here
+  except ImportError:
+    # but it's beeing moved. Next release (0.15.2) will be here
     # https://github.com/davidhalter/jedi/commit/3b4f2924648eafb9660caac9030b20beb50a83bb
     from jedi.inference import sys_path as jedi_inference_sys_path  # pylint: disable=import-error,unused-import,no-name-in-module
-_ = jedi_inference_sys_path.discover_buildout_paths  # make sure it's here
+  _ = jedi_inference_sys_path.discover_buildout_paths  # make sure it's here
 
-
-def dont_discover_buildout_paths(*args, **kw):
+  def dont_discover_buildout_paths(*args, **kw):
     return set()
 
+  jedi_inference_sys_path.discover_buildout_paths = dont_discover_buildout_paths
+  from jedi.api import project as jedi_api_project
+  jedi_api_project.discover_buildout_paths = dont_discover_buildout_paths
 
-jedi_inference_sys_path.discover_buildout_paths = dont_discover_buildout_paths
-from jedi.api import project as jedi_api_project
-jedi_api_project.discover_buildout_paths = dont_discover_buildout_paths
+from jedi.evaluate.context.instance import TreeInstance
+from jedi.evaluate.gradual.typing import InstanceWrapper
+from jedi.evaluate.lazy_context import LazyKnownContexts
+from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
 
-try:
-  # for local types annotations in this component
-  from erp5.portal_type import ERP5Site  # pylint: disable=unused-import,no-name-in-module
-except ImportError:
-  pass
+if typing.TYPE_CHECKING:
+  import erp5.portal_type.ERP5Site  # pylint: disable=unused-import,no-name-in-module,import-error
 
 
 def executeJediXXX(callback, context, arguments):
-  from jedi.evaluate.base_context import ContextSet
-
   # XXX function for relaodability
   def call():
     return callback(context, arguments=arguments)
 
-  from jedi.evaluate.context.instance import TreeInstance
-  from jedi.evaluate.gradual.typing import InstanceWrapper
-  from jedi.evaluate.lazy_context import LazyKnownContexts
-  from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS
-
   def makeFilterFunc(class_from_portal_type, arguments):
     def filter_func(val):
       if isinstance(val, TreeInstance) and val.tree_node.type == 'classdef':
@@ -71,6 +67,7 @@ def executeJediXXX(callback, context, arguments):
         for wrapped in val.iterate():
           if filter_func(wrapped):
             return True
+        return False
         annotation_classes = val.gather_annotation_classes()
         #import pdb; pdb.set_trace()
         return val.gather_annotation_classes().filter(filter_func)
@@ -87,6 +84,9 @@ def executeJediXXX(callback, context, arguments):
     if not arguments.argument_node:
       return call()  # no portal_type, we'll use what's defined in the stub
 
+    original = call()
+    #logger.info('re-evaluating %s ...', original)
+
     # look for a "portal_type=" argument
     for arg_name, arg_value in arguments.unpack():
       if arg_name == 'portal_type':
@@ -120,15 +120,11 @@ def executeJediXXX(callback, context, arguments):
 def makeERP5Plugin():
   logger.info('making erp5 plugin')
 
-  from jedi.evaluate.base_context import ContextSet
-  from jedi.parser_utils import get_cached_code_lines
-  from StringIO import StringIO
-
   class JediERP5Plugin(object):
     _cache = {}
 
     def _getPortalObject(self):  # XXX needed ?
-      # type: () -> ERP5Site
+      # type: () -> erp5.portal_type.ERP5Site
       from Products.ERP5.ERP5Site import getSite
       from Products.ERP5Type.Globals import get_request
       from ZPublisher.BaseRequest import RequestContainer
@@ -142,53 +138,10 @@ def makeERP5Plugin():
       logger.info("JediERP5Plugin registering execute")
 
       def wrapper(context, arguments):
+        # XXX call an external function that will be reloaded
         from erp5.component.extension.Jedi import executeJediXXX as _execute
         return _execute(callback, context, arguments)
 
-        def call():
-          return callback(context, arguments=arguments)
-
-        # methods returning portal types
-        if context.is_function():
-          # and 1 or context.get_function_execution(
-          #).function_context.name.string_name == 'newContent':
-          if not arguments.argument_node:
-            return call(
-            )  # no portal_type, we'll use what's defined in the stub
-
-          # look for a "portal_type=" argument
-          for arg_name, arg_value in arguments.unpack():
-            if arg_name == 'portal_type':
-              try:
-                portal_type = iter(arg_value.infer()).next().get_safe_value()
-              except Exception:
-                logger.exception("error infering")
-                continue
-              if not isinstance(portal_type, str):
-                continue
-              logger.info(
-                  'ahah portal_type based method with portal type=%s ...',
-                  portal_type)
-              # XXX this is really horrible
-              class_from_portal_type = portal_type.replace(' ', '') + '@'
-              original = call()
-              if 0:
-                filtered = ContextSet.from_sets({s for s in original})
-              import pdb
-              pdb.set_trace()
-              original._set = frozenset({
-                  x for x in original._set if class_from_portal_type in str(x)
-              })
-              logger.info(
-                  'portal_type based method, returning\n   %s instead of\n   %s',
-                  original,
-                  call())
-              #import pdb; pdb.set_trace()
-              return original
-
-        # methods returning List of portal types
-        # methods returning List of Brain of portal types
-        return call()
       return wrapper
 
   return JediERP5Plugin()
@@ -236,12 +189,20 @@ _TYPE_MAP = {
 
 
 def _label(definition):
+  # type: (jedi.api.classes.Completion,) -> str
+  if definition.type == 'param':
+    return '{}='.format(definition.name)
   if definition.type in ('function', 'method') and hasattr(definition,
                                                            'params'):
     params = ', '.join([param.name for param in definition.params])
     return '{}({})'.format(definition.name, params)
   return definition.name
 
+def _insertText(definition):
+  # type: (jedi.api.classes.Completion,) -> str
+  if definition.type == 'param':
+    return '{}='.format(definition.name)
+  return definition.name
 
 def _detail(definition):
   try:
@@ -264,21 +225,22 @@ def _format_docstring(docstring):
 
 
 def _format_completion(d):
+  # type: (jedi.api.classes.Completion,) -> typing.Dict[str, str]
   completion = {
       'label': _label(d),
       '_kind': _TYPE_MAP.get(d.type),
       'detail': _detail(d),
       'documentation': _format_docstring(d.docstring()),
       'sortText': _sort_text(d),
-      'insertText': d.name
+      'insertText': _insertText(d),
   }
   return completion
 
 
 def _guessType(name, context_type=None):
   """guess the type of python script parameters based on naming conventions.
-
   """
+  # TODO: `state_change` arguments for workflow scripts
   name = name.split('=')[
       0]  # support also assigned names ( like REQUEST=None in params)
   if name == 'context' and context_type:
@@ -286,7 +248,7 @@ def _guessType(name, context_type=None):
   if name in (
       'context',
       'container',):
-    return 'ERP5Site'
+    return 'erp5.portal_type.ERP5Site'
   if name == 'script':
     return 'Products.PythonScripts.PythonScript'
   if name == 'REQUEST':
@@ -298,7 +260,7 @@ def _guessType(name, context_type=None):
 
 # Jedi is not thread safe
 import Products.ERP5Type.Utils
-jedi_lock = getattr(Products.ERP5Type.Utils, 'jedi_lock', None)
+jedi_lock = getattr(Products.ERP5Type.Utils, 'jedi_lock', None) # type: RLock
 if jedi_lock is None:
   logger.critical("There was no lock, making a new one")
   jedi_lock = Products.ERP5Type.Utils.jedi_lock = RLock()
@@ -310,6 +272,8 @@ def ERP5Site_getPythonSourceCodeCompletionList(self, data, REQUEST=None):
   """
   portal = self.getPortalObject()
   logger.debug('jedi get lock %s (%s)', jedi_lock, id(jedi_lock))
+  if not jedi_lock.acquire(False):
+    raise RuntimeError('jedi is locked')
   with jedi_lock:
 
     # register our erp5 plugin
@@ -323,6 +287,7 @@ def ERP5Site_getPythonSourceCodeCompletionList(self, data, REQUEST=None):
 
     # data contains the code, the bound names and the script params. From this
     # we reconstruct a function that can be checked
+
     def indent(text):
       return ''.join(("  " + line) for line in text.splitlines(True))
 
@@ -347,9 +312,10 @@ def ERP5Site_getPythonSourceCodeCompletionList(self, data, REQUEST=None):
               "context_type %s has no portal type, using ERP5Site",
               context_type)
           context_type = None
+        else:
+          context_type = 'erp5.portal_type.{}'.format(context_type)
 
-      imports = "from erp5.portal_type import {}; import Products.ERP5Type.Core.Folder; import ZPublisher.HTTPRequest; import Products.PythonScripts".format(
-          context_type or 'ERP5Site')
+      imports = "import erp5.portal_type; import Products.ERP5Type.Core.Folder; import ZPublisher.HTTPRequest; import Products.PythonScripts"
       type_annotation = "  #  type: ({}) -> None".format(
           ', '.join(
               [_guessType(part, context_type) for part in signature_parts]))
@@ -771,7 +737,6 @@ def TypeInformation_getStub(self):
   # everything can use ERP5Site_ skins
   imports.add('from erp5.skins_tool import ERP5Site as skins_tool_ERP5Site')
   base_classes.append('skins_tool_ERP5Site')
-
   base_classes.append(prefixed_class_name)
 
   class_template = textwrap.dedent(
@@ -865,7 +830,7 @@ def PropertySheet_getStub(self):
           docstring=safe_docstring("ahaha cool :)")))
 
   def _getPythonTypeFromPropertySheetType(prop):
-    # type: (erp5.portal_type.StandardProperty) -> str
+    # type: (erp5.portal_type.StandardProperty,) -> str
     property_sheet_type = prop.getElementaryType()
     if property_sheet_type in ('content', 'object'):
       # TODO
@@ -883,10 +848,20 @@ def PropertySheet_getStub(self):
         'float': 'float',
         'text': 'str',
     }.get(property_sheet_type, 'Any')
-    if prop.isMultivalued():
+    if prop.isMultivalued() \
+          and property_sheet_type not in ('lines', 'token'):
+       # XXX see Resource/p_variation_base_category_property, we can have multivalued lines properties
       return 'List[{}]'.format(mapped_type)
     return mapped_type
 
+  def _isMultiValuedProperty(prop):
+    # type: (erp5.portal_type.StandardProperty,) -> str
+    """If this is a multi valued property, we have to generate list accessor.
+    """
+    if prop.isMultivalued():
+      return True
+    return prop.getElementaryType() in ('lines', 'tokens')
+
   from Products.ERP5Type.Utils import convertToUpperCase
   from Products.ERP5Type.Utils import evaluateExpressionFromString
   from Products.ERP5Type.Utils import createExpressionContext
@@ -908,8 +883,10 @@ def PropertySheet_getStub(self):
                   property_url=prop.absolute_url()))
       methods.append(
           method_template_template.format(
-              method_name='get{}'.format(
-                  convertToUpperCase(prop.getReference())),
+              method_name='get{}{}'.format(
+                  convertToUpperCase(prop.getReference()),
+                  'List' if _isMultiValuedProperty(prop) else '',
+              ),
               method_args='self',
               return_type=_getPythonTypeFromPropertySheetType(prop),
               docstring=docstring))
@@ -923,8 +900,10 @@ def PropertySheet_getStub(self):
                 docstring=docstring))
       methods.append(
           method_template_template.format(
-              method_name='set{}'.format(
-                  convertToUpperCase(prop.getReference())),
+              method_name='set{}{}'.format(
+                  convertToUpperCase(prop.getReference()),
+                  'List' if _isMultiValuedProperty(prop) else '',
+              ),
               method_args='self, value:{}'.format(
                   _getPythonTypeFromPropertySheetType(prop)),
               return_type='None',
@@ -1029,11 +1008,8 @@ def PropertySheet_getStub(self):
   )
 
 
-from Products.ERP5.ERP5Site import ERP5Site  # pylint: disable=unused-import
-
-
 def ERP5Site_getPortalStub(self):
-  # type: (ERP5Site) -> str
+  # type: (erp5.portal_type.ERP5Site,) -> erp5.portal_type.ERP5Site
 
   module_stub_template = textwrap.dedent(
       '''
@@ -1078,9 +1054,10 @@ def ERP5Site_getPortalStub(self):
 
   return textwrap.dedent(
       '''
-      from Products.ERP5Site.ERP5Site import ERP5Site as ERP5Site_parent_ERP5Site
+      from Products.ERP5.ERP5Site import ERP5Site as ERP5Site_parent_ERP5Site
       from erp5.skins_tool import ERP5Site as skins_tool_ERP5Site
       from erp5.skins_tool import Base as skins_tool_Base
+      
       class ERP5Site(ERP5Site_parent_ERP5Site, skins_tool_ERP5Site, skins_tool_Base):
         {}
         def getPortalObject(self):
@@ -1089,6 +1066,7 @@ def ERP5Site_getPortalStub(self):
 
 
 def ERP5Site_dumpModuleCode(self, component_or_script=None):
+  # type: (erp5.portal_type.ERP5Site,) -> None
   """Save code in filesystem for jedi to use it.
 
   Generate stubs for erp5.* dynamic modules and copy the in-ZODB modules
@@ -1098,10 +1076,7 @@ def ERP5Site_dumpModuleCode(self, component_or_script=None):
     if not os.path.exists(path):
       os.mkdir(path, 0o700)
 
-  # this import is for type annotation, but pylint does not understand this.
-  import erp5.portal_type  # pylint: disable=unused-variable
-
-  portal = self.getPortalObject()  # type: erp5.portal_type.ERP5Site
+  portal = self.getPortalObject()
   module_dir = '/tmp/ahaha/erp5/'  # TODO
   mkdir_p(module_dir)
 
@@ -1139,8 +1114,19 @@ def ERP5Site_dumpModuleCode(self, component_or_script=None):
                 ),
                 'w',
             ) as type_information_f:
-              type_information_f.write(
-                  ti.TypeInformation_getStub().encode('utf-8'))
+              try:
+                stub_code = ti.TypeInformation_getStub().encode('utf-8')
+              except Exception as e:
+                logger.exception("Could not generate code for %s", ti.getId())
+                stub_code = """class {class_name}:\n  {error}""".format(
+                    class_name=class_name,
+                    error=safe_docstring("Error trying to create {}: {} {}".format(
+                        ti.getId(),
+                        e.__class__,
+                        e
+                    ))
+                )
+              type_information_f.write(stub_code)
 
           # portal type groups ( useful ? used in Simulation Tool only )
           portal_types_by_group = defaultdict(list)

bt5/erp5_monaco_editor/ExtensionTemplateItem/portal_components/extension.erp5.Jedi.xml

@@ -45,14 +45,7 @@
         <item>
             <key> <string>text_content_warning_message</string> </key>
             <value>
-              <tuple>
-                <string>W: 61,  2: Reimport \'ContextSet\' (imported line 52) (reimported)</string>
-                <string>W: 79,  8: Unused variable \'annotation_classes\' (unused-variable)</string>
-                <string>W:156,  8: Unreachable code (unreachable)</string>
-                <string>W:184, 16: Unused variable \'filtered\' (unused-variable)</string>
-                <string>W:132,  2: Unused variable \'get_cached_code_lines\' (unused-variable)</string>
-                <string>W:133,  2: Unused variable \'StringIO\' (unused-variable)</string>
-              </tuple>
+              <tuple/>
             </value>
         </item>
         <item>

product/ERP5/bootstrap/erp5_core/ExtensionTemplateItem/portal_components/extension.erp5.PythonCodeUtils.py

+import time
 import json
+import logging
 from Products.ERP5Type.Utils import checkPythonSourceCode
 
+logger = logging.getLogger('extension.erp5.PythonCodeUtils')
+
+
 def checkPythonSourceCodeAsJSON(self, data, REQUEST=None):
   """
   Check Python source suitable for Source Code Editor and return a JSON object
   """
+  import json
 
   # XXX data is encoded as json, because jQuery serialize lists as []
   if isinstance(data, basestring):
@@ -16,6 +22,7 @@ def checkPythonSourceCodeAsJSON(self, data, REQUEST=None):
     return ''.join(("  " + line) for line in text.splitlines(True))
 
   # don't show 'undefined-variable' errors for {Python,Workflow} Script parameters
+  script_name = data.get('script_name') or 'unknown.py'
   is_script = 'bound_names' in data
   if is_script:
     signature_parts = data['bound_names']
@@ -30,7 +37,85 @@ def checkPythonSourceCodeAsJSON(self, data, REQUEST=None):
   else:
     body = data['code']
 
-  message_list = checkPythonSourceCode(body.encode('utf8'), data.get('portal_type'))
+  start = time.time()
+  code = body.encode('utf8')
+
+  import pyflakes.api
+  import pyflakes.reporter
+  import pyflakes.messages
+
+  class Reporter(pyflakes.reporter.Reporter):
+    def __init__(self):  # pylint: disable=super-init-not-called
+      self.message_list = []
+
+    def addMessage(self, row, column, level, text):
+      self.message_list.append(
+          dict(row=row, column=column, type=level, text=text))
+
+    def flake(self, message):
+      # type: (pyflakes.messages.Message,) -> None
+      self.addMessage(
+          row=message.lineno,
+          column=message.col,
+          text=message.message % (message.message_args),
+          level='W')
+
+    def syntaxError(self, filename, msg, lineno, offset, text):
+      self.addMessage(
+          row=lineno,
+          column=offset,
+          text='SyntaxError: {}'.format(text),
+          level='E')
+
+    def unexpectedError(self, filename, msg):
+      # TODO: extend interface to have range and in this case whole range is wrong ?
+      # or use parse with python in that case ?
+      # repro: function(a="b", c)
+      self.addMessage(
+          row=0, column=0, text='Unexpected Error: {}'.format(msg), level='E')
+
+  start = time.time()
+  reporter = Reporter()
+  pyflakes.api.check(code, script_name, reporter)
+  logger.info(
+      'pyflake checked %d lines in %.2f',
+      len(code.splitlines()),
+      time.time() - start
+    )
+
+  message_list = reporter.message_list
+
+  import lib2to3.refactor
+  import lib2to3.pgen2.parse
+  refactoring_tool = lib2to3.refactor.RefactoringTool(fixer_names=('lib2to3.fixes.fix_except', ))
+  old_code = code.decode('utf-8')
+  try:
+    new_code = unicode(refactoring_tool.refactor_string(old_code, script_name))
+  except lib2to3.pgen2.parse.ParseError as e:
+    message, (row, column) = e.context
+    message_list.append(
+          dict(row=row, column=column, type='E', text=message))
+  else:
+    if new_code != old_code:
+      i = 0
+      for new_line, old_line in zip(new_code.splitlines(), old_code.splitlines()):
+        i += 1
+        print ('new_line', new_line, 'old_line', old_line)
+        if new_line != old_line:
+          message_list.append(
+            dict(row=i, column=0, type='W', text=u'-{}\n+{}'.format(old_line, new_line)))
+
+#  import pdb; pdb.set_trace()
+  pylint_message_list = []
+  if 0:
+    start = time.time()
+    pylint_message_list = checkPythonSourceCode(code, data.get('portal_type'))
+    logger.info(
+      'pylint checked %d lines in %.2f',
+      len(code.splitlines()),
+      time.time() - start
+    )
+
   for message_dict in message_list:
     if is_script:
       message_dict['row'] = message_dict['row'] - 2