Commit 08b0e6f1 authored by Stefan Behnel's avatar Stefan Behnel

join UnicodeNode sequences in JoinedStrNode at compile time

parent 541ff4d0
......@@ -7,6 +7,7 @@ cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
error=object, warning=object, copy=object, _unicode=object)
import copy
import itertools
from . import PyrexTypes
from . import Naming
......@@ -350,15 +351,24 @@ class PostParse(ScopeTrackingTransform):
def visit_JoinedStrNode(self, node):
"""
Clean up after the parser by discarding empty Unicode strings.
Empty or single-value join lists are not uncommon because f-string
format specs are always parsed into JoinedStrNodes.
Clean up after the parser by discarding empty Unicode strings and merging
substring sequences. Empty or single-value join lists are not uncommon
because f-string format specs are always parsed into JoinedStrNodes.
"""
self.visitchildren(node)
unicode_node = ExprNodes.UnicodeNode
values = []
for value in node.values:
if isinstance(value, ExprNodes.UnicodeNode) and not value.value:
continue # ignore empty Unicode strings
values.append(value)
for is_unode_group, substrings in itertools.groupby(node.values, lambda v: isinstance(v, unicode_node)):
if is_unode_group:
substrings = list(substrings)
unode = substrings[0]
if len(substrings) > 1:
unode.value = EncodedString(u''.join(node.value for node in substrings))
# ignore empty Unicode strings
if unode.value:
values.append(unode)
else:
values.extend(substrings)
if not values:
node = ExprNodes.UnicodeNode(node.pos, value=EncodedString(''))
......@@ -366,7 +376,6 @@ class PostParse(ScopeTrackingTransform):
node = values[0]
else:
node.values = values
self.visitchildren(node)
return node
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment