Merge pull request #2583 from cjgibson/2508-slicing-fixes

Support two-element slicing with literal None

Cython/Compiler/ExprNodes.py

@@ -4794,10 +4794,50 @@ class SliceIndexNode(ExprNode):
                     step=none_node
                 ).analyse_types(env)
         else:
+            from .UtilNodes import EvalWithTempExprNode, ResultRefNode
             c_int = PyrexTypes.c_py_ssize_t_type
             if self.start:
+                if self.start.type.is_pyobject:
+                    start_ref = ResultRefNode(self.start)
+                    start_expr = CondExprNode(
+                        self.start.pos,
+                        true_val = IntNode(
+                            self.start.pos,
+                            value = '0',
+                            constant_result = 0
+                        ),
+                        false_val = start_ref,
+                        test = PrimaryCmpNode(
+                            self.start.pos,
+                            operand1 = start_ref,
+                            operator = 'is',
+                            operand2 = NoneNode(self.start.pos),
+                        )
+                    )
+                    start_expr = start_expr.analyse_types(env)
+                    start_expr = start_expr.coerce_to(c_int, env)
+                    self.start = EvalWithTempExprNode(start_ref, start_expr)
                 self.start = self.start.coerce_to(c_int, env)
             if self.stop:
+                if self.stop.type.is_pyobject:
+                    stop_ref = ResultRefNode(self.stop)
+                    stop_expr = CondExprNode(
+                        self.stop.pos,
+                        true_val = IntNode(
+                            self.stop.pos,
+                            value = 'PY_SSIZE_T_MAX'
+                        ),
+                        false_val = stop_ref,
+                        test = PrimaryCmpNode(
+                            self.stop.pos,
+                            operand1 = stop_ref,
+                            operator = 'is',
+                            operand2 = NoneNode(self.stop.pos),
+                        )
+                    )
+                    stop_expr = stop_expr.analyse_types(env)
+                    stop_expr = stop_expr.coerce_to(c_int, env)
+                    self.stop = EvalWithTempExprNode(stop_ref, stop_expr)
                 self.stop = self.stop.coerce_to(c_int, env)
         self.is_temp = 1
         return self

tests/run/slice2.pyx

 # mode: run
-# tag: slicing
+# tag: list, slice, slicing
 
 def test_full(seq):
     """
@@ -18,21 +18,24 @@ def test_full(seq):
 
 def test_start(seq, start):
     """
-    >>> test_start([1,2,3,4], 2)
+    >>> l = [1,2,3,4]
+    >>> test_start(l, 2)
     [3, 4]
-    >>> test_start([1,2,3,4], 3)
+    >>> test_start(l, 3)
     [4]
-    >>> test_start([1,2,3,4], 4)
+    >>> test_start(l, 4)
     []
-    >>> test_start([1,2,3,4], 8)
+    >>> test_start(l, 8)
     []
-    >>> test_start([1,2,3,4], -3)
+    >>> test_start(l, -3)
     [2, 3, 4]
-    >>> test_start([1,2,3,4], -4)
+    >>> test_start(l, -4)
+    [1, 2, 3, 4]
+    >>> test_start(l, -8)
     [1, 2, 3, 4]
-    >>> test_start([1,2,3,4], -8)
+    >>> test_start(l, 0)
     [1, 2, 3, 4]
-    >>> test_start([1,2,3,4], 0)
+    >>> test_start(l, None)
     [1, 2, 3, 4]
     >>> try: test_start(42, 2, 3)
     ... except TypeError: pass
@@ -42,24 +45,52 @@ def test_start(seq, start):
 
 def test_stop(seq, stop):
     """
-    >>> test_stop([1,2,3,4], 3)
+    >>> l = [1,2,3,4]
+    >>> test_stop(l, 3)
     [1, 2, 3]
-    >>> test_stop([1,2,3,4], -1)
+    >>> test_stop(l, -1)
     [1, 2, 3]
-    >>> test_stop([1,2,3,4], -3)
+    >>> test_stop(l, -3)
     [1]
-    >>> test_stop([1,2,3,4], -4)
+    >>> test_stop(l, -4)
     []
-    >>> test_stop([1,2,3,4], -8)
+    >>> test_stop(l, -8)
     []
-    >>> test_stop([1,2,3,4], 0)
+    >>> test_stop(l, 0)
     []
+    >>> test_stop(l, None)
+    [1, 2, 3, 4]
     >>> try: test_stop(42, 3)
     ... except TypeError: pass
     """
     obj = seq[:stop]
     return obj
 
+def test_step(seq, step):
+    """
+    >>> l = [1,2,3,4]
+    >>> test_step(l, -1)
+    [4, 3, 2, 1]
+    >>> test_step(l, 1)
+    [1, 2, 3, 4]
+    >>> test_step(l, 2)
+    [1, 3]
+    >>> test_step(l, 3)
+    [1, 4]
+    >>> test_step(l, -3)
+    [4, 1]
+    >>> test_step(l, None)
+    [1, 2, 3, 4]
+    >>> try: test_step(l, 0)
+    ... except ValueError: pass
+    ...
+    >>> try: test_step(42, 0)
+    ... except TypeError: pass
+    ...
+    """
+    obj = seq[::step]
+    return obj
+
 def test_start_and_stop(seq, start, stop):
     """
     >>> l = [1,2,3,4]
@@ -67,12 +98,38 @@ def test_start_and_stop(seq, start, stop):
     [3]
     >>> test_start_and_stop(l, -3, -1)
     [2, 3]
+    >>> test_start_and_stop(l, None, None)
+    [1, 2, 3, 4]
     >>> try: test_start_and_stop(42, 2, 3)
     ... except TypeError: pass
     """
     obj = seq[start:stop]
     return obj
 
+def test_start_stop_and_step(seq, start, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> test_start_stop_and_step(l, 0, 5, 1)
+    [1, 2, 3, 4, 5]
+    >>> test_start_stop_and_step(l, 5, -1, -1)
+    []
+    >>> test_start_stop_and_step(l, 5, None, -1)
+    [5, 4, 3, 2, 1]
+    >>> test_start_stop_and_step(l, 2, 5, 2)
+    [3, 5]
+    >>> test_start_stop_and_step(l, -100, 100, 1)
+    [1, 2, 3, 4, 5]
+    >>> test_start_stop_and_step(l, None, None, None)
+    [1, 2, 3, 4, 5]
+    >>> try: test_start_stop_and_step(l, None, None, 0)
+    ... except ValueError: pass
+    ... 
+    >>> try: test_start_stop_and_step(42, 1, 2, 3)
+    ... except TypeError: pass
+    """
+    obj = seq[start:stop:step]
+    return obj
+
 class A(object):
     pass
 

tests/run/typed_slice.pyx

 # mode: run
-# tag: list, tuple, slice
+# tag: list, tuple, slice, slicing
 
 def slice_list(list l, int start, int stop):
     """
@@ -203,3 +203,381 @@ def slice_charp_repeat(py_string_arg):
     cdef bytes slice_val = s[1:6]
     s = slice_val
     return s[1:3].decode(u'ASCII')
+
+# Readers will find the common boilerplate in the tests below:
+#     >>> l = [1,2,3,4,5]
+#     >>> t = tuple(l)
+#     >>> b = ''.join(map(str, l)).encode('ASCII')
+#     >>> u = b.decode('ASCII')
+#     >>> o = (l, t, b, u)
+#     >>> n = ('list', 'tuple', 'bytes', 'unicode')
+#     >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+#     >>> r = lambda i, *a: '%s[%s] -> %s' % (n[i], ':'.join(map(repr, a)), FUNCTION_NAME(o[i], *a))
+# Originally, this was planned to be a basic iteration over
+#   the various object types contained within the slicable fused
+#   type, but Python2 -> Python3 semantics changed the class names
+#   and string representations used for raw bytes and unicode.
+# As a result, we dynamically adjust the printed string output
+#   for each test in order to ensure consistent results when running
+#   both Python2 and Python3.
+
+ctypedef fused slicable:
+    list
+    tuple
+    bytes
+    unicode
+
+def slice_fused_type_start(slicable seq, start):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[%r:] -> %s' % (n[i], s, p(slice_fused_type_start(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (0, len(l) - 1, len(l), -1, -len(l), None):
+    ...         print(r(i, s))
+    ... 
+    list[0:] -> [1, 2, 3, 4, 5]
+    list[4:] -> [5]
+    list[5:] -> []
+    list[-1:] -> [5]
+    list[-5:] -> [1, 2, 3, 4, 5]
+    list[None:] -> [1, 2, 3, 4, 5]
+    tuple[0:] -> (1, 2, 3, 4, 5)
+    tuple[4:] -> (5,)
+    tuple[5:] -> ()
+    tuple[-1:] -> (5,)
+    tuple[-5:] -> (1, 2, 3, 4, 5)
+    tuple[None:] -> (1, 2, 3, 4, 5)
+    bytes[0:] -> 12345
+    bytes[4:] -> 5
+    bytes[5:] -> 
+    bytes[-1:] -> 5
+    bytes[-5:] -> 12345
+    bytes[None:] -> 12345
+    unicode[0:] -> 12345
+    unicode[4:] -> 5
+    unicode[5:] -> 
+    unicode[-1:] -> 5
+    unicode[-5:] -> 12345
+    unicode[None:] -> 12345
+    """
+    obj = seq[start:]
+    return obj
+
+def slice_fused_type_stop(slicable seq, stop):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[:%r] -> %s' % (n[i], s, p(slice_fused_type_stop(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (0, len(l) - 1, len(l), -1, -len(l), None):
+    ...         print(r(i, s))
+    ... 
+    list[:0] -> []
+    list[:4] -> [1, 2, 3, 4]
+    list[:5] -> [1, 2, 3, 4, 5]
+    list[:-1] -> [1, 2, 3, 4]
+    list[:-5] -> []
+    list[:None] -> [1, 2, 3, 4, 5]
+    tuple[:0] -> ()
+    tuple[:4] -> (1, 2, 3, 4)
+    tuple[:5] -> (1, 2, 3, 4, 5)
+    tuple[:-1] -> (1, 2, 3, 4)
+    tuple[:-5] -> ()
+    tuple[:None] -> (1, 2, 3, 4, 5)
+    bytes[:0] -> 
+    bytes[:4] -> 1234
+    bytes[:5] -> 12345
+    bytes[:-1] -> 1234
+    bytes[:-5] -> 
+    bytes[:None] -> 12345
+    unicode[:0] -> 
+    unicode[:4] -> 1234
+    unicode[:5] -> 12345
+    unicode[:-1] -> 1234
+    unicode[:-5] -> 
+    unicode[:None] -> 12345
+    """
+    obj = seq[:stop]
+    return obj
+
+def slice_fused_type_start_and_stop(slicable seq, start, stop):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, t, s: '%s[%r:%r] -> %s' % (n[i], t, s, p(slice_fused_type_start_and_stop(o[i], t, s)))
+    >>> for i in range(len(o)): 
+    ...     for start, stop in ((0, len(l)), (0, None), (None, len(l)),
+    ...                         (-len(l), 0), (1, 0), (0, 1)):
+    ...         print(r(i, start, stop))
+    ... 
+    list[0:5] -> [1, 2, 3, 4, 5]
+    list[0:None] -> [1, 2, 3, 4, 5]
+    list[None:5] -> [1, 2, 3, 4, 5]
+    list[-5:0] -> []
+    list[1:0] -> []
+    list[0:1] -> [1]
+    tuple[0:5] -> (1, 2, 3, 4, 5)
+    tuple[0:None] -> (1, 2, 3, 4, 5)
+    tuple[None:5] -> (1, 2, 3, 4, 5)
+    tuple[-5:0] -> ()
+    tuple[1:0] -> ()
+    tuple[0:1] -> (1,)
+    bytes[0:5] -> 12345
+    bytes[0:None] -> 12345
+    bytes[None:5] -> 12345
+    bytes[-5:0] -> 
+    bytes[1:0] -> 
+    bytes[0:1] -> 1
+    unicode[0:5] -> 12345
+    unicode[0:None] -> 12345
+    unicode[None:5] -> 12345
+    unicode[-5:0] -> 
+    unicode[1:0] -> 
+    unicode[0:1] -> 1
+    """
+    obj = seq[start:stop]
+    return obj
+
+def slice_fused_type_step(slicable seq, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[::%r] -> %s' % (n[i], s, p(slice_fused_type_step(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (1, -1, 2, -3, 5, -5, None):
+    ...         print(r(i, s))
+    ... 
+    list[::1] -> [1, 2, 3, 4, 5]
+    list[::-1] -> [5, 4, 3, 2, 1]
+    list[::2] -> [1, 3, 5]
+    list[::-3] -> [5, 2]
+    list[::5] -> [1]
+    list[::-5] -> [5]
+    list[::None] -> [1, 2, 3, 4, 5]
+    tuple[::1] -> (1, 2, 3, 4, 5)
+    tuple[::-1] -> (5, 4, 3, 2, 1)
+    tuple[::2] -> (1, 3, 5)
+    tuple[::-3] -> (5, 2)
+    tuple[::5] -> (1,)
+    tuple[::-5] -> (5,)
+    tuple[::None] -> (1, 2, 3, 4, 5)
+    bytes[::1] -> 12345
+    bytes[::-1] -> 54321
+    bytes[::2] -> 135
+    bytes[::-3] -> 52
+    bytes[::5] -> 1
+    bytes[::-5] -> 5
+    bytes[::None] -> 12345
+    unicode[::1] -> 12345
+    unicode[::-1] -> 54321
+    unicode[::2] -> 135
+    unicode[::-3] -> 52
+    unicode[::5] -> 1
+    unicode[::-5] -> 5
+    unicode[::None] -> 12345
+    >>> for v in o:
+    ...     try: slice_fused_type_step(v, 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_step(v, v)
+    ...     except TypeError: pass
+    """
+    obj = seq[::step]
+    return obj
+
+def slice_fused_type_start_and_step(slicable seq, start, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t: '%s[%r::%r] -> %s' % (n[i], s, t, p(slice_fused_type_start_and_step(o[i], s, t)))
+    >>> for i in range(len(o)):
+    ...     for start, step in ((0, 1), (0, -1), (1, 1), (1, -1),
+    ...                         (None, 1), (None, -1), (None, None),
+    ...                         (1, 2), (len(l), -2), (len(l), len(l))):
+    ...         print(r(i, start, step))
+    ... 
+    list[0::1] -> [1, 2, 3, 4, 5]
+    list[0::-1] -> [1]
+    list[1::1] -> [2, 3, 4, 5]
+    list[1::-1] -> [2, 1]
+    list[None::1] -> [1, 2, 3, 4, 5]
+    list[None::-1] -> [5, 4, 3, 2, 1]
+    list[None::None] -> [1, 2, 3, 4, 5]
+    list[1::2] -> [2, 4]
+    list[5::-2] -> [5, 3, 1]
+    list[5::5] -> []
+    tuple[0::1] -> (1, 2, 3, 4, 5)
+    tuple[0::-1] -> (1,)
+    tuple[1::1] -> (2, 3, 4, 5)
+    tuple[1::-1] -> (2, 1)
+    tuple[None::1] -> (1, 2, 3, 4, 5)
+    tuple[None::-1] -> (5, 4, 3, 2, 1)
+    tuple[None::None] -> (1, 2, 3, 4, 5)
+    tuple[1::2] -> (2, 4)
+    tuple[5::-2] -> (5, 3, 1)
+    tuple[5::5] -> ()
+    bytes[0::1] -> 12345
+    bytes[0::-1] -> 1
+    bytes[1::1] -> 2345
+    bytes[1::-1] -> 21
+    bytes[None::1] -> 12345
+    bytes[None::-1] -> 54321
+    bytes[None::None] -> 12345
+    bytes[1::2] -> 24
+    bytes[5::-2] -> 531
+    bytes[5::5] -> 
+    unicode[0::1] -> 12345
+    unicode[0::-1] -> 1
+    unicode[1::1] -> 2345
+    unicode[1::-1] -> 21
+    unicode[None::1] -> 12345
+    unicode[None::-1] -> 54321
+    unicode[None::None] -> 12345
+    unicode[1::2] -> 24
+    unicode[5::-2] -> 531
+    unicode[5::5] -> 
+    >>> for o in (l, t, b):
+    ...     try: slice_fused_type_start_and_step(o, 0, 0)
+    ...     except ValueError: pass
+    """
+    obj = seq[start::step]
+    return obj
+
+def slice_fused_type_stop_and_step(slicable seq, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t: '%s[:%r:%r] -> %s' % (n[i], s, t, p(slice_fused_type_stop_and_step(o[i], s, t)))
+    >>> for i in range(len(o)):
+    ...     for stop, step in ((len(l), 1), (len(l), None), (None, 1),
+    ...                        (len(l), -1), (len(l) - 1, 2), (len(l), -2),
+    ...                        (len(l), len(l))):
+    ...         print(r(i, stop, step))
+    ... 
+    list[:5:1] -> [1, 2, 3, 4, 5]
+    list[:5:None] -> [1, 2, 3, 4, 5]
+    list[:None:1] -> [1, 2, 3, 4, 5]
+    list[:5:-1] -> []
+    list[:4:2] -> [1, 3]
+    list[:5:-2] -> []
+    list[:5:5] -> [1]
+    tuple[:5:1] -> (1, 2, 3, 4, 5)
+    tuple[:5:None] -> (1, 2, 3, 4, 5)
+    tuple[:None:1] -> (1, 2, 3, 4, 5)
+    tuple[:5:-1] -> ()
+    tuple[:4:2] -> (1, 3)
+    tuple[:5:-2] -> ()
+    tuple[:5:5] -> (1,)
+    bytes[:5:1] -> 12345
+    bytes[:5:None] -> 12345
+    bytes[:None:1] -> 12345
+    bytes[:5:-1] -> 
+    bytes[:4:2] -> 13
+    bytes[:5:-2] -> 
+    bytes[:5:5] -> 1
+    unicode[:5:1] -> 12345
+    unicode[:5:None] -> 12345
+    unicode[:None:1] -> 12345
+    unicode[:5:-1] -> 
+    unicode[:4:2] -> 13
+    unicode[:5:-2] -> 
+    unicode[:5:5] -> 1
+    >>> for v in o:
+    ...     try: slice_fused_type_stop_and_step(v, len(l), 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_stop_and_step(v, len(l), v)
+    ...     except TypeError: pass
+    """
+    obj = seq[:stop:step]
+    return obj
+
+def slice_fused_type_all(slicable seq, start, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t, e: '%s[%r:%r:%r] -> %s' % (n[i], s, t, e, p(slice_fused_type_all(o[i], s, t, e)))
+    >>> for i in range(len(o)):
+    ...     for args in ((0, len(l), 1), (len(l), 0, -1), (None, len(l), 1),
+    ...                  (len(l), None, -1), (-len(l), len(l), None), (None, None, None),
+    ...                  (1, 3, 2), (len(l), 1, -3), (len(l), 0, 1)):
+    ...         print(r(i, *args))
+    ... 
+    list[0:5:1] -> [1, 2, 3, 4, 5]
+    list[5:0:-1] -> [5, 4, 3, 2]
+    list[None:5:1] -> [1, 2, 3, 4, 5]
+    list[5:None:-1] -> [5, 4, 3, 2, 1]
+    list[-5:5:None] -> [1, 2, 3, 4, 5]
+    list[None:None:None] -> [1, 2, 3, 4, 5]
+    list[1:3:2] -> [2]
+    list[5:1:-3] -> [5]
+    list[5:0:1] -> []
+    tuple[0:5:1] -> (1, 2, 3, 4, 5)
+    tuple[5:0:-1] -> (5, 4, 3, 2)
+    tuple[None:5:1] -> (1, 2, 3, 4, 5)
+    tuple[5:None:-1] -> (5, 4, 3, 2, 1)
+    tuple[-5:5:None] -> (1, 2, 3, 4, 5)
+    tuple[None:None:None] -> (1, 2, 3, 4, 5)
+    tuple[1:3:2] -> (2,)
+    tuple[5:1:-3] -> (5,)
+    tuple[5:0:1] -> ()
+    bytes[0:5:1] -> 12345
+    bytes[5:0:-1] -> 5432
+    bytes[None:5:1] -> 12345
+    bytes[5:None:-1] -> 54321
+    bytes[-5:5:None] -> 12345
+    bytes[None:None:None] -> 12345
+    bytes[1:3:2] -> 2
+    bytes[5:1:-3] -> 5
+    bytes[5:0:1] -> 
+    unicode[0:5:1] -> 12345
+    unicode[5:0:-1] -> 5432
+    unicode[None:5:1] -> 12345
+    unicode[5:None:-1] -> 54321
+    unicode[-5:5:None] -> 12345
+    unicode[None:None:None] -> 12345
+    unicode[1:3:2] -> 2
+    unicode[5:1:-3] -> 5
+    unicode[5:0:1] -> 
+    >>> for v in o:
+    ...     try: slice_fused_type_stop_and_step(v, len(l), 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_stop_and_step(v, len(l), v)
+    ...     except TypeError: pass
+    """
+    obj = seq[start:stop:step]
+    return obj