Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation...

Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation is still needed, I'll open an issue for that.

Include/Python-ast.h

@@ -201,9 +201,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
                   Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
                   Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
-                  Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
-                  Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
-                  Name_kind=25, List_kind=26, Tuple_kind=27};
+                  FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
+                  NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24,
+                  Subscript_kind=25, Starred_kind=26, Name_kind=27,
+                  List_kind=28, Tuple_kind=29};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -296,6 +297,16 @@ struct _expr {
             string s;
         } Str;
         
+        struct {
+            expr_ty value;
+            int conversion;
+            expr_ty format_spec;
+        } FormattedValue;
+        
+        struct {
+            asdl_seq *values;
+        } JoinedStr;
+        
         struct {
             bytes s;
         } Bytes;
@@ -543,6 +554,12 @@ expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
 expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
 expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
+                           int lineno, int col_offset, PyArena *arena);
+#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3)
+expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
+                      *arena);
 #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
 expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
 #define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)

Misc/NEWS

@@ -19,6 +19,11 @@ Core and Builtins
   argument list of a function declaration.  For example, "def f(*, a =
   3,): pass" is now legal. Patch from Mark Dickinson.
 
+- Issue #24965: Implement PEP 498 "Literal String Interpolation". This
+  allows you to embed expressions inside f-strings, which are
+  converted to normal strings at run time. Given x=3, then
+  f'value={x}' == 'value=3'. Patch by Eric V. Smith.
+
 Library
 -------
 

Parser/Python.asdl

@@ -71,6 +71,8 @@ module Python
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | FormattedValue(expr value, int? conversion, expr? format_spec)
+         | JoinedStr(expr* values)
          | Bytes(bytes s)
          | NameConstant(singleton value)
          | Ellipsis

Parser/tokenizer.c

@@ -1477,17 +1477,19 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     nonascii = 0;
     if (is_potential_identifier_start(c)) {
         /* Process b"", r"", u"", br"" and rb"" */
-        int saw_b = 0, saw_r = 0, saw_u = 0;
+        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
         while (1) {
-            if (!(saw_b || saw_u) && (c == 'b' || c == 'B'))
+            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
                 saw_b = 1;
             /* Since this is a backwards compatibility support literal we don't
                want to support it in arbitrary order like byte literals. */
-            else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U'))
+            else if (!(saw_b || saw_u || saw_r || saw_f) && (c == 'u' || c == 'U'))
                 saw_u = 1;
             /* ur"" and ru"" are not supported */
             else if (!(saw_r || saw_u) && (c == 'r' || c == 'R'))
                 saw_r = 1;
+            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F'))
+                saw_f = 1;
             else
                 break;
             c = tok_nextc(tok);

Python/Python-ast.c

@@ -285,6 +285,18 @@ _Py_IDENTIFIER(s);
 static char *Str_fields[]={
     "s",
 };
+static PyTypeObject *FormattedValue_type;
+_Py_IDENTIFIER(conversion);
+_Py_IDENTIFIER(format_spec);
+static char *FormattedValue_fields[]={
+    "value",
+    "conversion",
+    "format_spec",
+};
+static PyTypeObject *JoinedStr_type;
+static char *JoinedStr_fields[]={
+    "values",
+};
 static PyTypeObject *Bytes_type;
 static char *Bytes_fields[]={
     "s",
@@ -917,6 +929,11 @@ static int init_types(void)
     if (!Num_type) return 0;
     Str_type = make_type("Str", expr_type, Str_fields, 1);
     if (!Str_type) return 0;
+    FormattedValue_type = make_type("FormattedValue", expr_type,
+                                    FormattedValue_fields, 3);
+    if (!FormattedValue_type) return 0;
+    JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
+    if (!JoinedStr_type) return 0;
     Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
     if (!Bytes_type) return 0;
     NameConstant_type = make_type("NameConstant", expr_type,
@@ -2062,6 +2079,42 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
     return p;
 }
 
+expr_ty
+FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
+               int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    if (!value) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field value is required for FormattedValue");
+        return NULL;
+    }
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = FormattedValue_kind;
+    p->v.FormattedValue.value = value;
+    p->v.FormattedValue.conversion = conversion;
+    p->v.FormattedValue.format_spec = format_spec;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
+expr_ty
+JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = JoinedStr_kind;
+    p->v.JoinedStr.values = values;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
 expr_ty
 Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
 {
@@ -3161,6 +3214,34 @@ ast2obj_expr(void* _o)
             goto failed;
         Py_DECREF(value);
         break;
+    case FormattedValue_kind:
+        result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_expr(o->v.FormattedValue.value);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_int(o->v.FormattedValue.conversion);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        value = ast2obj_expr(o->v.FormattedValue.format_spec);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
+    case JoinedStr_kind:
+        result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Bytes_kind:
         result = PyType_GenericNew(Bytes_type, NULL, NULL);
         if (!result) goto failed;
@@ -6022,6 +6103,86 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        expr_ty value;
+        int conversion;
+        expr_ty format_spec;
+
+        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_value);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &value, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue");
+            return 1;
+        }
+        if (exists_not_none(obj, &PyId_conversion)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_conversion);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_int(tmp, &conversion, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            conversion = 0;
+        }
+        if (exists_not_none(obj, &PyId_format_spec)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_format_spec);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &format_spec, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            format_spec = NULL;
+        }
+        *out = FormattedValue(value, conversion, format_spec, lineno,
+                              col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        asdl_seq* values;
+
+        if (_PyObject_HasAttrId(obj, &PyId_values)) {
+            int res;
+            Py_ssize_t len;
+            Py_ssize_t i;
+            tmp = _PyObject_GetAttrId(obj, &PyId_values);
+            if (tmp == NULL) goto failed;
+            if (!PyList_Check(tmp)) {
+                PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+                goto failed;
+            }
+            len = PyList_GET_SIZE(tmp);
+            values = _Py_asdl_seq_new(len, arena);
+            if (values == NULL) goto failed;
+            for (i = 0; i < len; i++) {
+                expr_ty value;
+                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena);
+                if (res != 0) goto failed;
+                asdl_seq_SET(values, i, value);
+            }
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr");
+            return 1;
+        }
+        *out = JoinedStr(values, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type);
     if (isinstance == -1) {
         return 1;
@@ -7319,6 +7480,10 @@ PyInit__ast(void)
     if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "FormattedValue",
+        (PyObject*)FormattedValue_type) < 0) return NULL;
+    if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
         NULL;
     if (PyDict_SetItemString(d, "NameConstant", (PyObject*)NameConstant_type) <

Python/compile.c

@@ -731,6 +731,7 @@ compiler_set_qualname(struct compiler *c)
     return 1;
 }
 
+
 /* Allocate a new block and return a pointer to it.
    Returns NULL on error.
 */
@@ -3209,6 +3210,117 @@ compiler_call(struct compiler *c, expr_ty e)
                                 e->v.Call.keywords);
 }
 
+static int
+compiler_joined_str(struct compiler *c, expr_ty e)
+{
+    /* Concatenate parts of a string using ''.join(parts). There are
+       probably better ways of doing this.
+
+       This is used for constructs like "'x=' f'{42}'", which have to
+       be evaluated at compile time. */
+
+    static PyObject *empty_string;
+    static PyObject *join_string;
+
+    if (!empty_string) {
+        empty_string = PyUnicode_FromString("");
+        if (!empty_string)
+            return 0;
+    }
+    if (!join_string) {
+        join_string = PyUnicode_FromString("join");
+        if (!join_string)
+            return 0;
+    }
+
+    ADDOP_O(c, LOAD_CONST, empty_string, consts);
+    ADDOP_NAME(c, LOAD_ATTR, join_string, names);
+    VISIT_SEQ(c, expr, e->v.JoinedStr.values);
+    ADDOP_I(c, BUILD_LIST, asdl_seq_LEN(e->v.JoinedStr.values));
+    ADDOP_I(c, CALL_FUNCTION, 1);
+    return 1;
+}
+
+/* Note that this code uses the builtin functions format(), str(),
+   repr(), and ascii(). You can break this code, or make it do odd
+   things, by redefining those functions. */
+static int
+compiler_formatted_value(struct compiler *c, expr_ty e)
+{
+    PyObject *conversion_name = NULL;
+
+    static PyObject *format_string;
+    static PyObject *str_string;
+    static PyObject *repr_string;
+    static PyObject *ascii_string;
+
+    if (!format_string) {
+        format_string = PyUnicode_InternFromString("format");
+        if (!format_string)
+            return 0;
+    }
+
+    if (!str_string) {
+        str_string = PyUnicode_InternFromString("str");
+        if (!str_string)
+            return 0;
+    }
+
+    if (!repr_string) {
+        repr_string = PyUnicode_InternFromString("repr");
+        if (!repr_string)
+            return 0;
+    }
+    if (!ascii_string) {
+        ascii_string = PyUnicode_InternFromString("ascii");
+        if (!ascii_string)
+            return 0;
+    }
+
+    ADDOP_NAME(c, LOAD_GLOBAL, format_string, names);
+
+    /* If needed, convert via str, repr, or ascii. */
+    if (e->v.FormattedValue.conversion != -1) {
+        switch (e->v.FormattedValue.conversion) {
+        case 's':
+            conversion_name = str_string;
+            break;
+        case 'r':
+            conversion_name = repr_string;
+            break;
+        case 'a':
+            conversion_name = ascii_string;
+            break;
+        default:
+            PyErr_SetString(PyExc_SystemError,
+                            "Unrecognized conversion character");
+            return 0;
+        }
+        ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names);
+    }
+
+    /* Evaluate the value. */
+    VISIT(c, expr, e->v.FormattedValue.value);
+
+    /* If needed, convert via str, repr, or ascii. */
+    if (conversion_name) {
+        /* Call the function we previously pushed. */
+        ADDOP_I(c, CALL_FUNCTION, 1);
+    }
+
+    /* If we have a format spec, use format(value, format_spec). Otherwise,
+       use the single argument form. */
+    if (e->v.FormattedValue.format_spec) {
+        VISIT(c, expr, e->v.FormattedValue.format_spec);
+        ADDOP_I(c, CALL_FUNCTION, 2);
+    } else {
+        /* No format spec specified, call format(value). */
+        ADDOP_I(c, CALL_FUNCTION, 1);
+    }
+
+    return 1;
+}
+
 /* shared code between compiler_call and compiler_class */
 static int
 compiler_call_helper(struct compiler *c,
@@ -3878,6 +3990,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case Str_kind:
         ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts);
         break;
+    case JoinedStr_kind:
+        return compiler_joined_str(c, e);
+    case FormattedValue_kind:
+        return compiler_formatted_value(c, e);
     case Bytes_kind:
         ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts);
         break;
@@ -4784,4 +4900,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
 {
     return PyAST_CompileEx(mod, filename, flags, -1, arena);
 }
-

Python/symtable.c

@@ -1439,6 +1439,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
         VISIT_SEQ(st, expr, e->v.Call.args);
         VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
         break;
+    case FormattedValue_kind:
+        VISIT(st, expr, e->v.FormattedValue.value);
+        if (e->v.FormattedValue.format_spec)
+            VISIT(st, expr, e->v.FormattedValue.format_spec);
+        break;
+    case JoinedStr_kind:
+        VISIT_SEQ(st, expr, e->v.JoinedStr.values);
+        break;
     case Num_kind:
     case Str_kind:
     case Bytes_kind: