merge heads

Doc/c-api/typeobj.rst

@@ -745,6 +745,11 @@ type objects) *must* have the :attr:`ob_size` field.
    This field is not inherited by subtypes (though the attributes defined in here
    are inherited through a different mechanism).
 
+   .. warning::
+
+      It is not safe to use :c:func:`PyDict_SetItem` on or otherwise modify
+      :attr:`tp_dict` with the dictionary C-API.
+
 
 .. c:member:: descrgetfunc PyTypeObject.tp_descr_get
 

Include/Python-ast.h

@@ -271,7 +271,7 @@ struct _expr {
                 } Str;
                 
                 struct {
-                        string s;
+                        bytes s;
                 } Bytes;
                 
                 struct {
@@ -501,7 +501,7 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
 expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
 #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
-expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena);
+expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
 #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
 expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)

Include/asdl.h

@@ -3,6 +3,7 @@
 
 typedef PyObject * identifier;
 typedef PyObject * string;
+typedef PyObject * bytes;
 typedef PyObject * object;
 
 /* It would be nice if the code generated by asdl_c.py was completely

Include/ast.h

@@ -4,6 +4,7 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(int) PyAST_Validate(mod_ty);
 PyAPI_FUNC(mod_ty) PyAST_FromNode(
     const node *n,
     PyCompilerFlags *flags,

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.3 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #12575: Validate user-generated AST before it is compiled.
+
 - Make type(None), type(Ellipsis), and type(NotImplemented) callable. They
   return the respective singleton instances.
 

Parser/Python.asdl

--- ASDL's four builtin types are identifier, int, string, object
+-- ASDL's five builtin types are identifier, int, string, bytes, object
 
 module Python
 {
@@ -67,7 +67,7 @@ module Python
 			 expr? starargs, expr? kwargs)
 	     | Num(object n) -- a number as a PyObject.
 	     | Str(string s) -- need to specify raw, unicode, etc?
-	     | Bytes(string s)
+	     | Bytes(bytes s)
 	     | Ellipsis
 	     -- other literals? bools?
 

Parser/asdl.py

@@ -228,7 +228,7 @@ class ASDLParser(spark.GenericParser, object):
         " field ::= Id ? "
         return Field(type[0], opt=True)
 
-builtin_types = ("identifier", "string", "int", "bool", "object")
+builtin_types = ("identifier", "string", "bytes", "int", "object")
 
 # below is a collection of classes to capture the AST of an AST :-)
 # not sure if any of the methods are useful yet, but I'm adding them

Parser/asdl_c.py

@@ -776,6 +776,7 @@ static PyObject* ast2obj_object(void *o)
 }
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
+#define ast2obj_bytes ast2obj_object
 
 static PyObject* ast2obj_int(long b)
 {
@@ -813,6 +814,15 @@ static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
     return obj2ast_object(obj, out, arena);
 }
 
+static int obj2ast_bytes(PyObject* obj, PyObject** out, PyArena* arena)
+{
+    if (!PyBytes_CheckExact(obj)) {
+        PyErr_SetString(PyExc_TypeError, "AST bytes must be of type bytes");
+        return 1;
+    }
+    return obj2ast_object(obj, out, arena);
+}
+
 static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
 {
     int i;

Python/Python-ast.c

@@ -573,6 +573,7 @@ static PyObject* ast2obj_object(void *o)
 }
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
+#define ast2obj_bytes ast2obj_object
 
 static PyObject* ast2obj_int(long b)
 {
@@ -610,6 +611,15 @@ static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
     return obj2ast_object(obj, out, arena);
 }
 
+static int obj2ast_bytes(PyObject* obj, PyObject** out, PyArena* arena)
+{
+    if (!PyBytes_CheckExact(obj)) {
+        PyErr_SetString(PyExc_TypeError, "AST bytes must be of type bytes");
+        return 1;
+    }
+    return obj2ast_object(obj, out, arena);
+}
+
 static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
 {
     int i;
@@ -1773,7 +1783,7 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
-Bytes(string s, int lineno, int col_offset, PyArena *arena)
+Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!s) {
@@ -2804,7 +2814,7 @@ ast2obj_expr(void* _o)
         case Bytes_kind:
                 result = PyType_GenericNew(Bytes_type, NULL, NULL);
                 if (!result) goto failed;
-                value = ast2obj_string(o->v.Bytes.s);
+                value = ast2obj_bytes(o->v.Bytes.s);
                 if (!value) goto failed;
                 if (PyObject_SetAttrString(result, "s", value) == -1)
                         goto failed;
@@ -5509,13 +5519,13 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                 return 1;
         }
         if (isinstance) {
-                string s;
+                bytes s;
 
                 if (PyObject_HasAttrString(obj, "s")) {
                         int res;
                         tmp = PyObject_GetAttrString(obj, "s");
                         if (tmp == NULL) goto failed;
-                        res = obj2ast_string(tmp, &s, arena);
+                        res = obj2ast_bytes(tmp, &s, arena);
                         if (res != 0) goto failed;
                         Py_XDECREF(tmp);
                         tmp = NULL;

Python/bltinmodule.c

@@ -604,6 +604,10 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
                 PyArena_Free(arena);
                 goto error;
             }
+            if (!PyAST_Validate(mod)) {
+                PyArena_Free(arena);
+                goto error;
+            }
             result = (PyObject*)PyAST_CompileEx(mod, filename,
                                                 &cf, optimize, arena);
             PyArena_Free(arena);