parse integer literals differently based on the type of source file: .pyx vs. .py, Py2 vs. Py3 mode

Cython/Compiler/ExprNodes.py

@@ -800,9 +800,11 @@ class IntNode(ConstNode):
 
     # unsigned     "" or "U"
     # longness     "" or "L" or "LL"
+    # is_c_literal   True/False/None   creator considers this a C integer literal
 
     unsigned = ""
     longness = ""
+    is_c_literal = None # unknown
 
     def __init__(self, pos, **kwds):
         ExprNode.__init__(self, pos, **kwds)
@@ -815,7 +817,10 @@ class IntNode(ConstNode):
                 self.calculate_constant_result()
             except ValueError:
                 pass
-        if self.constant_result in (constant_value_not_set, not_a_constant) or \
+        # we ignore 'is_c_literal = True' and instead map signed 32bit
+        # integers as C long values
+        if self.is_c_literal or \
+               self.constant_result in (constant_value_not_set, not_a_constant) or \
                self.unsigned or self.longness == 'LL':
             # clearly a C literal
             rank = (self.longness == 'LL') and 2 or 1
@@ -844,17 +849,18 @@ class IntNode(ConstNode):
             else:
                 return FloatNode(self.pos, value=self.value, type=dst_type,
                                  constant_result=not_a_constant)
-        node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
-                       type = dst_type, unsigned=self.unsigned, longness=self.longness)
         if dst_type.is_numeric and not dst_type.is_complex:
             node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
-                           type = dst_type, unsigned=self.unsigned, longness=self.longness)
+                           type = dst_type, is_c_literal = True,
+                           unsigned=self.unsigned, longness=self.longness)
             return node
         elif dst_type.is_pyobject:
             node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
-                           type = PyrexTypes.py_object_type, unsigned=self.unsigned, longness=self.longness)
+                           type = PyrexTypes.py_object_type, is_c_literal = False,
+                           unsigned=self.unsigned, longness=self.longness)
         else:
-            # not setting the type here!
+            # FIXME: not setting the type here to keep it working with
+            # complex numbers. Should they be special cased?
             node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
                            unsigned=self.unsigned, longness=self.longness)
         # We still need to perform normal coerce_to processing on the

Cython/Compiler/Parsing.pxd

 # We declare all of these here to type the first argument.
 
+cimport cython
 from Cython.Compiler.Scanning cimport PyrexScanner
 
 
@@ -46,6 +47,8 @@ cpdef expect_ellipsis(PyrexScanner s)
 cpdef make_slice_nodes(pos, subscripts)
 cpdef make_slice_node(pos, start, stop = *, step = *)
 cpdef p_atom(PyrexScanner s)
+@cython.locals(value=unicode)
+cpdef p_int_literal(PyrexScanner s)
 cpdef p_name(PyrexScanner s, name)
 cpdef p_cat_string_literal(PyrexScanner s)
 cpdef p_opt_string_literal(PyrexScanner s, required_type=*)

Cython/Compiler/Parsing.py

@@ -583,20 +583,7 @@ def p_atom(s):
         expect_ellipsis(s)
         return ExprNodes.EllipsisNode(pos)
     elif sy == 'INT':
-        value = s.systring
-        s.next()
-        unsigned = ""
-        longness = ""
-        while value[-1] in "UuLl":
-            if value[-1] in "Ll":
-                longness += "L"
-            else:
-                unsigned += "U"
-            value = value[:-1]
-        return ExprNodes.IntNode(pos, 
-                                 value = value,
-                                 unsigned = unsigned,
-                                 longness = longness)
+        return p_int_literal(s)
     elif sy == 'FLOAT':
         value = s.systring
         s.next()
@@ -631,6 +618,37 @@ def p_atom(s):
     else:
         s.error("Expected an identifier or literal")
 
+def p_int_literal(s):
+    pos = s.position()
+    value = s.systring
+    s.next()
+    unsigned = ""
+    longness = ""
+    while value[-1] in u"UuLl":
+        if value[-1] in u"Ll":
+            longness += "L"
+        else:
+            unsigned += "U"
+        value = value[:-1]
+    # '3L' is ambiguous in Py2 but not in Py3.  '3U' and '3LL' are
+    # illegal in Py2 Python files.  All suffixes are illegal in Py3
+    # Python files.
+    is_c_literal = None
+    if unsigned:
+        is_c_literal = True
+    elif longness:
+        if longness == 'LL' or s.context.language_level >= 3:
+            is_c_literal = True
+    if s.in_python_file:
+        if is_c_literal:
+            error(pos, "illegal integer literal syntax in Python source file")
+        is_c_literal = False
+    return ExprNodes.IntNode(pos,
+                             is_c_literal = is_c_literal,
+                             value = value,
+                             unsigned = unsigned,
+                             longness = longness)
+
 def p_name(s, name):
     pos = s.position()
     if not s.compile_time_expr and name in s.compile_time_env:

tests/errors/e_int_literals_py2.py

+# cython: language_level=2
+
+def int_literals():
+    a = 1L  # ok
+    b = 10000000000000L # ok
+    c = 1UL
+    d = 10000000000000UL
+    e = 10000000000000LL
+
+
+_ERRORS = """
+6:8: illegal integer literal syntax in Python source file
+7:8: illegal integer literal syntax in Python source file
+8:8: illegal integer literal syntax in Python source file
+"""

tests/errors/e_int_literals_py3.py

+# cython: language_level=3
+
+def int_literals():
+    a = 1L
+    b = 10000000000000L
+    c = 1UL
+    d = 10000000000000UL
+    e = 10000000000000LL
+
+
+_ERRORS = """
+4:8: illegal integer literal syntax in Python source file
+5:8: illegal integer literal syntax in Python source file
+6:8: illegal integer literal syntax in Python source file
+7:8: illegal integer literal syntax in Python source file
+8:8: illegal integer literal syntax in Python source file
+"""

tests/run/cython3.pyx

@@ -106,3 +106,16 @@ def dict_iter(dict d):
     values = [ value for value in d.values() ]
     items = [ item for item in d.items() ]
     return keys, values, items
+
+def int_literals():
+    """
+    >>> int_literals()
+    long
+    long
+    unsigned long
+    unsigned long
+    """
+    print(cython.typeof(1L))
+    print(cython.typeof(10000000000000L))
+    print(cython.typeof(1UL))
+    print(cython.typeof(10000000000000UL))