Commit fe3a65f7 authored by Robert Bradshaw's avatar Robert Bradshaw

F strings (PEP 498)

parents 37534635 b9c66f41
...@@ -2954,6 +2954,111 @@ class RawCNameExprNode(ExprNode): ...@@ -2954,6 +2954,111 @@ class RawCNameExprNode(ExprNode):
pass pass
#-------------------------------------------------------------------
#
# F-strings
#
#-------------------------------------------------------------------
class JoinedStrNode(ExprNode):
# F-strings
#
# values [UnicodeNode|FormattedValueNode] Substrings of the f-string
#
type = unicode_type
is_temp = True
subexprs = ['values']
def analyse_types(self, env):
self.values = [v.analyse_types(env).coerce_to_pyobject(env) for v in self.values]
return self
def generate_evaluation_code(self, code):
code.mark_pos(self.pos)
num_items = len(self.values)
list_var = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
code.putln('%s = PyList_New(%s); %s' % (
list_var,
num_items,
code.error_goto_if_null(list_var, self.pos)))
code.put_gotref(list_var)
for i, node in enumerate(self.values):
node.generate_evaluation_code(code)
node.make_owned_reference(code)
code.put_giveref(node.py_result())
code.putln('PyList_SET_ITEM(%s, %s, %s);' % (list_var, i, node.py_result()))
node.generate_post_assignment_code(code)
node.free_temps(code)
code.mark_pos(self.pos)
self.allocate_temp_result(code)
code.putln('%s = PyUnicode_Join(%s, %s); %s' % (
self.result(),
Naming.empty_unicode,
list_var,
code.error_goto_if_null(self.py_result(), self.pos)))
code.put_gotref(self.py_result())
code.put_decref_clear(list_var, py_object_type)
code.funcstate.release_temp(list_var)
class FormattedValueNode(ExprNode):
# {}-delimited portions of an f-string
#
# value ExprNode The expression itself
# conversion_char str or None Type conversion (!s, !r, !a, or none)
# format_spec JoinedStrNode or None Format string passed to __format__
subexprs = ['value', 'format_spec']
type = py_object_type
is_temp = True
find_conversion_func = {
's': 'PyObject_Str',
'r': 'PyObject_Repr',
'a': 'PyObject_ASCII', # NOTE: Py3-only!
}.get
def analyse_types(self, env):
self.value = self.value.analyse_types(env).coerce_to_pyobject(env)
if self.format_spec:
self.format_spec = self.format_spec.analyse_types(env).coerce_to_pyobject(env)
return self
def generate_result_code(self, code):
value_result = self.value.py_result()
if self.format_spec:
format_func = '__Pyx_PyObject_Format'
format_spec = self.format_spec.py_result()
else:
# common case: expect simple Unicode pass-through if no format spec
format_func = '__Pyx_PyObject_FormatSimple'
format_spec = Naming.empty_unicode
if self.conversion_char:
fn = self.find_conversion_func(self.conversion_char)
assert fn is not None, "invalid conversion character found: '%s'" % self.conversion_char
value_result = '%s(%s)' % (fn, value_result)
code.globalstate.use_utility_code(UtilityCode.load_cached("PyObjectFormatAndDecref", "StringTools.c"))
format_func += 'AndDecref'
elif not self.format_spec:
code.globalstate.use_utility_code(UtilityCode.load_cached("PyObjectFormatSimple", "StringTools.c"))
else:
format_func = 'PyObject_Format'
code.putln("%s = %s(%s, %s); %s" % (
self.result(),
format_func,
value_result,
format_spec,
code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.py_result())
#------------------------------------------------------------------- #-------------------------------------------------------------------
# #
# Parallel nodes (cython.parallel.thread(savailable|id)) # Parallel nodes (cython.parallel.thread(savailable|id))
......
...@@ -7,7 +7,7 @@ from __future__ import absolute_import ...@@ -7,7 +7,7 @@ from __future__ import absolute_import
raw_prefixes = "rR" raw_prefixes = "rR"
bytes_prefixes = "bB" bytes_prefixes = "bB"
string_prefixes = "uU" + bytes_prefixes string_prefixes = "fFuU" + bytes_prefixes
char_prefixes = "cC" char_prefixes = "cC"
any_string_prefix = raw_prefixes + string_prefixes + char_prefixes any_string_prefix = raw_prefixes + string_prefixes + char_prefixes
IDENT = 'IDENT' IDENT = 'IDENT'
...@@ -40,8 +40,8 @@ def make_lexicon(): ...@@ -40,8 +40,8 @@ def make_lexicon():
fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
imagconst = (intconst | fltconst) + Any("jJ") imagconst = (intconst | fltconst) + Any("jJ")
beginstring = Opt(Any(string_prefixes) + Opt(Any(raw_prefixes)) | # invalid combinations of prefixes are caught in p_string_literal
Any(raw_prefixes) + Opt(Any(bytes_prefixes)) | beginstring = Opt(Rep(Any(string_prefixes + raw_prefixes)) |
Any(char_prefixes) Any(char_prefixes)
) + (Str("'") | Str('"') | Str("'''") | Str('"""')) ) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
two_oct = octdigit + octdigit two_oct = octdigit + octdigit
......
...@@ -695,6 +695,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -695,6 +695,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln('static PyObject *%s;' % Naming.builtins_cname) code.putln('static PyObject *%s;' % Naming.builtins_cname)
code.putln('static PyObject *%s;' % Naming.empty_tuple) code.putln('static PyObject *%s;' % Naming.empty_tuple)
code.putln('static PyObject *%s;' % Naming.empty_bytes) code.putln('static PyObject *%s;' % Naming.empty_bytes)
code.putln('static PyObject *%s;' % Naming.empty_unicode)
if Options.pre_import is not None: if Options.pre_import is not None:
code.putln('static PyObject *%s;' % Naming.preimport_cname) code.putln('static PyObject *%s;' % Naming.preimport_cname)
code.putln('static int %s;' % Naming.lineno_cname) code.putln('static int %s;' % Naming.lineno_cname)
...@@ -2117,6 +2118,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -2117,6 +2118,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos))) Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)))
code.putln("%s = PyBytes_FromStringAndSize(\"\", 0); %s" % ( code.putln("%s = PyBytes_FromStringAndSize(\"\", 0); %s" % (
Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos))) Naming.empty_bytes, code.error_goto_if_null(Naming.empty_bytes, self.pos)))
code.putln("%s = PyUnicode_FromStringAndSize(\"\", 0); %s" % (
Naming.empty_unicode, code.error_goto_if_null(Naming.empty_unicode, self.pos)))
for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'StopAsyncIteration'): for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'StopAsyncIteration'):
code.putln("#ifdef __Pyx_%s_USED" % ext_type) code.putln("#ifdef __Pyx_%s_USED" % ext_type)
......
...@@ -96,6 +96,7 @@ gilstate_cname = pyrex_prefix + "state" ...@@ -96,6 +96,7 @@ gilstate_cname = pyrex_prefix + "state"
skip_dispatch_cname = pyrex_prefix + "skip_dispatch" skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
empty_tuple = pyrex_prefix + "empty_tuple" empty_tuple = pyrex_prefix + "empty_tuple"
empty_bytes = pyrex_prefix + "empty_bytes" empty_bytes = pyrex_prefix + "empty_bytes"
empty_unicode = pyrex_prefix + "empty_unicode"
print_function = pyrex_prefix + "print" print_function = pyrex_prefix + "print"
print_function_kwargs = pyrex_prefix + "print_kwargs" print_function_kwargs = pyrex_prefix + "print_kwargs"
cleanup_cname = pyrex_prefix + "module_cleanup" cleanup_cname = pyrex_prefix + "module_cleanup"
......
...@@ -3,6 +3,7 @@ from __future__ import absolute_import ...@@ -3,6 +3,7 @@ from __future__ import absolute_import
import sys import sys
import copy import copy
import codecs import codecs
import itertools
from . import TypeSlots from . import TypeSlots
from .ExprNodes import not_a_constant from .ExprNodes import not_a_constant
...@@ -3934,6 +3935,44 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): ...@@ -3934,6 +3935,44 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations):
sequence_node.mult_factor = factor sequence_node.mult_factor = factor
return sequence_node return sequence_node
def visit_FormattedValueNode(self, node):
self.visitchildren(node)
if isinstance(node.format_spec, ExprNodes.UnicodeNode) and not node.format_spec.value:
node.format_spec = None
if node.format_spec is None and node.conversion_char is None and isinstance(node.value, ExprNodes.UnicodeNode):
return node.value
return node
def visit_JoinedStrNode(self, node):
"""
Clean up after the parser by discarding empty Unicode strings and merging
substring sequences. Empty or single-value join lists are not uncommon
because f-string format specs are always parsed into JoinedStrNodes.
"""
self.visitchildren(node)
unicode_node = ExprNodes.UnicodeNode
values = []
for is_unode_group, substrings in itertools.groupby(node.values, lambda v: isinstance(v, unicode_node)):
if is_unode_group:
substrings = list(substrings)
unode = substrings[0]
if len(substrings) > 1:
unode.value = EncodedString(u''.join(value.value for value in substrings))
# ignore empty Unicode strings
if unode.value:
values.append(unode)
else:
values.extend(substrings)
if not values:
node = ExprNodes.UnicodeNode(node.pos, value=EncodedString(''))
elif len(values) == 1:
node = values[0]
else:
node.values = values
return node
def visit_MergedDictNode(self, node): def visit_MergedDictNode(self, node):
"""Unpack **args in place if we can.""" """Unpack **args in place if we can."""
self.visitchildren(node) self.visitchildren(node)
......
...@@ -68,6 +68,10 @@ cdef p_opt_string_literal(PyrexScanner s, required_type=*) ...@@ -68,6 +68,10 @@ cdef p_opt_string_literal(PyrexScanner s, required_type=*)
cdef bint check_for_non_ascii_characters(unicode string) cdef bint check_for_non_ascii_characters(unicode string)
@cython.locals(systr=unicode, is_python3_source=bint, is_raw=bint) @cython.locals(systr=unicode, is_python3_source=bint, is_raw=bint)
cdef p_string_literal(PyrexScanner s, kind_override=*) cdef p_string_literal(PyrexScanner s, kind_override=*)
@cython.locals(i=Py_ssize_t, size=Py_ssize_t)
cdef list p_f_string(PyrexScanner s, unicode_value, pos)
@cython.locals(i=Py_ssize_t, size=Py_ssize_t, c=Py_UCS4, quote_char=Py_UCS4)
cdef tuple p_f_string_expr(PyrexScanner s, unicode_value, pos, Py_ssize_t starting_index)
cdef p_list_maker(PyrexScanner s) cdef p_list_maker(PyrexScanner s)
cdef p_comp_iter(PyrexScanner s, body) cdef p_comp_iter(PyrexScanner s, body)
cdef p_comp_for(PyrexScanner s, body) cdef p_comp_for(PyrexScanner s, body)
......
This diff is collapsed.
...@@ -134,6 +134,10 @@ ...@@ -134,6 +134,10 @@
#define __Pyx_PyString_Format(a, b) PyString_Format(a, b) #define __Pyx_PyString_Format(a, b) PyString_Format(a, b)
#endif #endif
#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII)
#define PyObject_ASCII(o) PyObject_Repr(o)
#endif
#if PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3
#define PyBaseString_Type PyUnicode_Type #define PyBaseString_Type PyUnicode_Type
#define PyStringObject PyUnicodeObject #define PyStringObject PyUnicodeObject
......
...@@ -811,3 +811,23 @@ static CYTHON_INLINE int __Pyx_PyByteArray_Append(PyObject* bytearray, int value ...@@ -811,3 +811,23 @@ static CYTHON_INLINE int __Pyx_PyByteArray_Append(PyObject* bytearray, int value
Py_DECREF(retval); Py_DECREF(retval);
return 0; return 0;
} }
//////////////////// PyObjectFormatSimple.proto ////////////////////
#define __Pyx_PyObject_FormatSimple(s, f) (likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) : PyObject_Format(s, f))
//////////////////// PyObjectFormatAndDecref.proto ////////////////////
#define __Pyx_PyObject_FormatSimpleAndDecref(s, f) \
((unlikely(!s) || likely(PyUnicode_CheckExact(s))) ? s : __Pyx_PyObject_FormatAndDecref(s, f))
static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f);
//////////////////// PyObjectFormatAndDecref ////////////////////
static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f) {
PyObject *result = PyObject_Format(s, f);
Py_DECREF(s);
return result;
}
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment