Merge pull request #411 from tjwei/jedi_0.9

Experimental support for jedi 0.9.0

Cython/Tests/TestJediTyper.py

@@ -66,9 +66,6 @@ class TestJediTyper(TransformTest):
             a = i + 1
         '''
         types = self._test(code)
-        if not types:
-            # old Jedi version
-            return
         self.assertIn((None, (1, 0)), types)
         variables = types.pop((None, (1, 0)))
         self.assertFalse(types)
@@ -87,7 +84,7 @@ class TestJediTyper(TransformTest):
         self.assertFalse(types)
         self.assertEqual({'a': set(['int']), 'i': set(['int'])}, variables)
 
-    def _test_conflicting_types_in_function(self):
+    def test_conflicting_types_in_function(self):
         code = '''\
         def func(a, b):
             print(a)
@@ -102,7 +99,7 @@ class TestJediTyper(TransformTest):
         self.assertIn(('func', (1, 0)), types)
         variables = types.pop(('func', (1, 0)))
         self.assertFalse(types)
-        self.assertEqual({'a': set(['int', 'str']), 'i': set(['int'])}, variables)
+        self.assertEqual({'a': set(['float', 'int', 'str']), 'b': set(['int'])}, variables)
 
     def _test_typing_function_char_loop(self):
         code = '''\
@@ -120,6 +117,92 @@ class TestJediTyper(TransformTest):
         self.assertFalse(types)
         self.assertEqual({'a': set(['int']), 'i': set(['int'])}, variables)
 
+    def test_typing_global_list(self):
+        code = '''\
+        a = [x for x in range(10)]
+        b = list(range(10))
+        c = a + b
+        d = [0]*10
+        '''
+        types = self._test(code)
+        self.assertIn((None, (1, 0)), types)
+        variables = types.pop((None, (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['list']), 'b': set(['list']), 'c': set(['list']), 'd': set(['list'])}, variables)
+        
+    def test_typing_function_list(self):
+        code = '''\
+        def func(x):
+            a = [[], []]
+            b = [0]* 10 + a
+            c = a[0]
+
+        print(func([0]*100))
+        '''
+        types = self._test(code)
+        self.assertIn(('func', (1, 0)), types)
+        variables = types.pop(('func', (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['list']), 'b': set(['list']), 'c': set(['list']), 'x': set(['list'])}, variables)
+
+    def test_typing_global_dict(self):
+        code = '''\
+        a = dict()
+        b = {i: i**2 for i in range(10)}
+        c = a        
+        '''
+        types = self._test(code)
+        self.assertIn((None, (1, 0)), types)
+        variables = types.pop((None, (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['dict']), 'b': set(['dict']), 'c': set(['dict'])}, variables)
+        
+    def test_typing_function_dict(self):
+        code = '''\
+        def func(x):
+            a = dict()
+            b = {i: i**2 for i in range(10)}
+            c = x
+
+        print(func({1:2, 'x':7}))
+        '''
+        types = self._test(code)
+        self.assertIn(('func', (1, 0)), types)
+        variables = types.pop(('func', (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['dict']), 'b': set(['dict']), 'c': set(['dict']), 'x': set(['dict'])}, variables)
+
+
+    def test_typing_global_set(self):
+        code = '''\
+        a = set()
+        # b = {i for i in range(10)} # jedi does not support set comprehension yet
+        c = a
+        d = {1,2,3}
+        e = a | b
+        '''
+        types = self._test(code)
+        self.assertIn((None, (1, 0)), types)
+        variables = types.pop((None, (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['set']), 'c': set(['set']), 'd': set(['set']), 'e': set(['set'])}, variables)
+        
+    def test_typing_function_set(self):
+        code = '''\
+        def func(x):
+            a = set()
+            # b = {i for i in range(10)} # jedi does not support set comprehension yet
+            c = a
+            d = a | b
+
+        print(func({1,2,3}))
+        '''
+        types = self._test(code)
+        self.assertIn(('func', (1, 0)), types)
+        variables = types.pop(('func', (1, 0)))
+        self.assertFalse(types)
+        self.assertEqual({'a': set(['set']), 'c': set(['set']), 'd': set(['set']), 'x': set(['set'])}, variables)
+
 
 class TestTypeInjection(TestJediTyper):
     """

Tools/jedi-typer.py

@@ -6,9 +6,12 @@ from __future__ import absolute_import
 
 from io import open
 from collections import defaultdict
+from itertools import chain
 
-from jedi import Script
-from jedi.parser.representation import Function, Module, Import
+import jedi
+from jedi.parser.tree import Module, ImportName
+from jedi.evaluate.representation import Function, Instance, Class
+from jedi.evaluate.iterable import ArrayMixin, GeneratorComprehension
 
 from Cython.Utils import open_source_file
 
@@ -26,33 +29,42 @@ def analyse(source_path=None, code=None):
     """
     if not source_path and code is None:
         raise ValueError("Either 'source_path' or 'code' is required.")
-    script = Script(source=code, path=source_path)
-    evaluator = script._evaluator
     scoped_names = {}
-    for statements in script._parser.module().used_names.values():
-        for statement in statements:
-            scope = statement.parent
-            while not isinstance(scope, (Function, Module)):
-                scope = scope.parent
-            # hack: work around current Jedi problem with global module variables
-            if not hasattr(scope, 'scope_names_generator'):
-                continue
-            statement_names = statement.get_defined_names()
-            if not statement_names:
-                continue
-            key = (None if isinstance(scope, Module) else str(scope.name), scope.start_pos)
-            try:
-                names = scoped_names[key]
-            except KeyError:
-                names = scoped_names[key] = defaultdict(set)
-            for name in statement_names:
-                for name_type in evaluator.find_types(scope, name):
-                    if isinstance(name_type, Import):
-                        type_name = 'object'
-                    else:
-                        type_name = name_type.name
-                    names[str(name)].add(type_name)
-
+    statement_iter = jedi.names(source=code, path=source_path, all_scopes=True)
+    
+    for statement in statement_iter:
+        parent = statement.parent()
+        scope = parent._definition
+        evaluator = statement._evaluator
+
+        # skip function/generator definitions, class definitions, and module imports
+        if any(isinstance(statement._definition, t) for t in [Function, Class, ImportName]):
+            continue
+        key = (None if isinstance(scope, Module) else str(parent.name), scope.start_pos)
+        try:
+            names = scoped_names[key]
+        except KeyError:
+            names = scoped_names[key] = defaultdict(set)
+                
+        position = statement.start_pos if statement.name in names else None
+
+        for name_type in evaluator.find_types(scope, statement.name, position=position ,search_global=True):
+            if isinstance(name_type, Instance):
+                if isinstance(name_type.base, Class):
+                    type_name = 'object'
+                else:
+                    type_name = name_type.base.obj.__name__
+            elif isinstance(name_type, ArrayMixin):
+                type_name = name_type.type
+            elif isinstance(name_type, GeneratorComprehension):
+                type_name = None
+            else:
+                try:
+                    type_name = type(name_type.obj).__name__                
+                except AttributeError as error:                    
+                    type_name = None
+            if type_name is not None:
+                names[str(statement.name)].add(type_name)
     return scoped_names
 
 
@@ -72,13 +84,14 @@ def inject_types(source_path, types, type_map=default_type_map, mode='python'):
         for line_no, line in enumerate(f, 1):
             if line_no in col_and_types_by_line:
                 col, scope, types = col_and_types_by_line[line_no]
-                types = ', '.join("%s='%s'" % (name, type_map.get(type_name, type_name))
-                                  for name, type_name in types)
-                if scope is None:
-                    type_decl = u'{indent}cython.declare({types})\n'
-                else:
-                    type_decl = u'{indent}@cython.locals({types})\n'
-                lines.append(type_decl.format(indent=' '*col, types=types))
+                if types:
+                    types = ', '.join("%s='%s'" % (name, type_map.get(type_name, type_name))
+                                    for name, type_name in types)
+                    if scope is None:
+                        type_decl = u'{indent}cython.declare({types})\n'
+                    else:
+                        type_decl = u'{indent}@cython.locals({types})\n'
+                    lines.append(type_decl.format(indent=' '*col, types=types))
             lines.append(line)
 
     return lines

runtests.py

@@ -1943,7 +1943,7 @@ def runtests(options, cmd_args, coverage=None):
 
     try:
         import jedi
-        if not ([0, 8, 1] <= list(map(int, re.findall('[0-9]+', jedi.__version__ or '0'))) < [0, 9]):
+        if not ([0, 9] <= list(map(int, re.findall('[0-9]+', jedi.__version__ or '0')))):
             raise ImportError
     except (ImportError, AttributeError, TypeError):
         exclude_selectors.append(RegExSelector('Jedi'))