Merge pull request #982 from kmod/repl2

Add support for using the CPython parser

CMakeLists.txt

@@ -287,9 +287,9 @@ endmacro()
 
 # tests         testname directory arguments
 add_pyston_test(defaults tests --order-by-mtime -t50)
-add_pyston_test(force_llvm tests -a=-n -a=-x -t50)
+add_pyston_test(force_llvm tests -a=-n -a=-X -t50)
 if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
-  add_pyston_test(max_compilation_tier tests -a=-O -a=-x -t50)
+  add_pyston_test(max_compilation_tier tests -a=-O -a=-X -t50)
 endif()
 add_pyston_test(defaults cpython --exit-code-only --skip-failing -t50)
 add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)

Makefile

@@ -779,7 +779,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1
 	@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t50 $(TEST_DIR)/cpython $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t600 $(TEST_DIR)/integration $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-X -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
 	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 
 .PHONY: run$1 dbg$1
@@ -825,6 +825,7 @@ perf_report:
 
 .PHONY: run run_% dbg_% debug_% perf_%
 run: run_dbg
+dbg: dbg_dbg
 run_%: run_dbg_%
 	@true
 dbg_%: dbg_dbg_%

from_cpython/CMakeLists.txt

@@ -94,21 +94,32 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
 
 # compile specified files in from_cpython/Python
 file(GLOB_RECURSE STDPYTHON_SRCS Python
+    asdl.c
+    ast.c
     dtoa.c
     formatter_string.c
     formatter_unicode.c
     getargs.c
+    graminit.c
     marshal.c
     mystrtoul.c
+    pyarena.c
     pyctype.c
     pystrtod.c
+    Python-ast.c
     random.c
     structmember.c
     )
 
 # compile specified files in from_cpython/Python
 file(GLOB_RECURSE STDPARSER_SRCS Parser
+    acceler.c
+    grammar1.c
     myreadline.c
+    node.c
+    parser.c
+    parsetok.c
+    tokenizer.c
     )
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing -DPy_BUILD_CORE")

from_cpython/Include/Python.h

@@ -41,9 +41,6 @@
 
 #include <assert.h>
 
-// CPython doesn't seem to include this but I'm not sure how they get the definition of 'bool':
-#include <stdbool.h>
-
 #include "pyport.h"
 
 #include "pymath.h"
@@ -133,8 +130,11 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 
 #include "abstract.h"
 
+#include "compile.h"
+
 #include "pyctype.h"
 #include "pystrtod.h"
+#include "pystrcmp.h"
 #include "dtoa.h"
 
 // directly from CPython:
@@ -185,9 +185,6 @@ extern PyTypeObject* Itertool_SafeDealloc_Types[];
 #define PyDoc_STRVAR(name, str) PyDoc_VAR(name) = PyDoc_STR(str)
 #define PyDoc_STR(str) str
 
-// This is in Python-ast.h in CPython, which we don't yet have:
-int PyAST_Check(PyObject* obj) PYSTON_NOEXCEPT;
-
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" void
 #else

from_cpython/Include/abstract.h

@@ -534,7 +534,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
     /* new buffer API */
 
      // Pyston change: made this a function
-    bool _PyObject_CheckBuffer(PyObject* obj) PYSTON_NOEXCEPT;
+    int _PyObject_CheckBuffer(PyObject* obj) PYSTON_NOEXCEPT;
 #define PyObject_CheckBuffer(obj) _PyObject_CheckBuffer((PyObject*)(obj))
 
     /* Return 1 if the getbuffer function is available, otherwise
@@ -847,7 +847,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
        */
 
      // Pyston change: made this a function:
-    PyAPI_FUNC(bool) _PyIndex_Check(PyObject* o) PYSTON_NOEXCEPT;
+    PyAPI_FUNC(int) _PyIndex_Check(PyObject* o) PYSTON_NOEXCEPT;
 #define PyIndex_Check(obj) _PyIndex_Check((PyObject*)(obj))
 #if 0
 #define PyIndex_Check(obj) \

from_cpython/Include/asdl.h

+#ifndef Py_ASDL_H
+#define Py_ASDL_H
+
+typedef PyObject * identifier;
+typedef PyObject * string;
+typedef PyObject * object;
+
+#ifndef __cplusplus
+typedef enum {false, true} bool;
+#endif
+
+/* It would be nice if the code generated by asdl_c.py was completely
+   independent of Python, but it is a goal the requires too much work
+   at this stage.  So, for example, I'll represent identifiers as
+   interned Python strings.
+*/
+
+/* XXX A sequence should be typed so that its use can be typechecked. */
+
+typedef struct {
+    int size;
+    void *elements[1];
+} asdl_seq;
+
+typedef struct {
+    int size;
+    int elements[1];
+} asdl_int_seq;
+
+asdl_seq *asdl_seq_new(int size, PyArena *arena);
+asdl_int_seq *asdl_int_seq_new(int size, PyArena *arena);
+
+#define asdl_seq_GET(S, I) (S)->elements[(I)]
+#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
+#ifdef Py_DEBUG
+#define asdl_seq_SET(S, I, V) { \
+        int _asdl_i = (I); \
+        assert((S) && _asdl_i < (S)->size); \
+        (S)->elements[_asdl_i] = (V); \
+}
+#else
+#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
+#endif
+
+#endif /* !Py_ASDL_H */

from_cpython/Include/ast.h

+#ifndef Py_AST_H
+#define Py_AST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
+				  const char *, PyArena *) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_AST_H */

from_cpython/Include/bitset.h

+
+#ifndef Py_BITSET_H
+#define Py_BITSET_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Bitset interface */
+
+#define BYTE		char
+
+typedef BYTE *bitset;
+
+bitset newbitset(int nbits);
+void delbitset(bitset bs);
+#define testbit(ss, ibit) (((ss)[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0)
+int addbit(bitset bs, int ibit); /* Returns 0 if already set */
+int samebitset(bitset bs1, bitset bs2, int nbits);
+void mergebitset(bitset bs1, bitset bs2, int nbits);
+
+#define BITSPERBYTE	(8*sizeof(BYTE))
+#define NBYTES(nbits)	(((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
+
+#define BIT2BYTE(ibit)	((ibit) / BITSPERBYTE)
+#define BIT2SHIFT(ibit)	((ibit) % BITSPERBYTE)
+#define BIT2MASK(ibit)	(1 << BIT2SHIFT(ibit))
+#define BYTE2BIT(ibyte)	((ibyte) * BITSPERBYTE)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_BITSET_H */

from_cpython/Include/errcode.h

+#ifndef Py_ERRCODE_H
+#define Py_ERRCODE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Error codes passed around between file input, tokenizer, parser and
+   interpreter.  This is necessary so we can turn them into Python
+   exceptions at a higher level.  Note that some errors have a
+   slightly different meaning when passed from the tokenizer to the
+   parser than when passed from the parser to the interpreter; e.g.
+   the parser only returns E_EOF when it hits EOF immediately, and it
+   never returns E_OK. */
+
+#define E_OK		10	/* No error */
+#define E_EOF		11	/* End Of File */
+#define E_INTR		12	/* Interrupted */
+#define E_TOKEN		13	/* Bad token */
+#define E_SYNTAX	14	/* Syntax error */
+#define E_NOMEM		15	/* Ran out of memory */
+#define E_DONE		16	/* Parsing complete */
+#define E_ERROR		17	/* Execution error */
+#define E_TABSPACE	18	/* Inconsistent mixing of tabs and spaces */
+#define E_OVERFLOW      19	/* Node had too many children */
+#define E_TOODEEP	20	/* Too many indentation levels */
+#define E_DEDENT	21	/* No matching outer block for dedent */
+#define E_DECODE	22	/* Error in decoding into Unicode */
+#define E_EOFS		23	/* EOF in triple-quoted string */
+#define E_EOLS		24	/* EOL in single-quoted string */
+#define E_LINECONT	25	/* Unexpected characters after a line continuation */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_ERRCODE_H */

from_cpython/Include/eval.h

+
+/* Interface to execute compiled code */
+
+#ifndef Py_EVAL_H
+#define Py_EVAL_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *);
+
+PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
+					PyObject *globals,
+					PyObject *locals,
+					PyObject **args, int argc,
+					PyObject **kwds, int kwdc,
+					PyObject **defs, int defc,
+					PyObject *closure);
+
+PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_EVAL_H */

from_cpython/Include/fileobject.h

@@ -63,6 +63,9 @@ PyAPI_FUNC(int) PyFile_SoftSpace(PyObject *, int) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *) PYSTON_NOEXCEPT;
 
+// Pyston temporary addition:
+PyAPI_FUNC(PyObject *) PyFile_GetEncoding(PyObject *) PYSTON_NOEXCEPT;
+
 /* The default encoding used by the platform file system APIs
    If non-NULL, this is different than the default encoding for strings
 */

from_cpython/Include/graminit.h

+/* Generated by Parser/pgen */
+
+#define single_input 256
+#define file_input 257
+#define eval_input 258
+#define decorator 259
+#define decorators 260
+#define decorated 261
+#define funcdef 262
+#define parameters 263
+#define varargslist 264
+#define fpdef 265
+#define fplist 266
+#define stmt 267
+#define simple_stmt 268
+#define small_stmt 269
+#define expr_stmt 270
+#define augassign 271
+#define print_stmt 272
+#define del_stmt 273
+#define pass_stmt 274
+#define flow_stmt 275
+#define break_stmt 276
+#define continue_stmt 277
+#define return_stmt 278
+#define yield_stmt 279
+#define raise_stmt 280
+#define import_stmt 281
+#define import_name 282
+#define import_from 283
+#define import_as_name 284
+#define dotted_as_name 285
+#define import_as_names 286
+#define dotted_as_names 287
+#define dotted_name 288
+#define global_stmt 289
+#define exec_stmt 290
+#define assert_stmt 291
+#define compound_stmt 292
+#define if_stmt 293
+#define while_stmt 294
+#define for_stmt 295
+#define try_stmt 296
+#define with_stmt 297
+#define with_item 298
+#define except_clause 299
+#define suite 300
+#define testlist_safe 301
+#define old_test 302
+#define old_lambdef 303
+#define test 304
+#define or_test 305
+#define and_test 306
+#define not_test 307
+#define comparison 308
+#define comp_op 309
+#define expr 310
+#define xor_expr 311
+#define and_expr 312
+#define shift_expr 313
+#define arith_expr 314
+#define term 315
+#define factor 316
+#define power 317
+#define atom 318
+#define listmaker 319
+#define testlist_comp 320
+#define lambdef 321
+#define trailer 322
+#define subscriptlist 323
+#define subscript 324
+#define sliceop 325
+#define exprlist 326
+#define testlist 327
+#define dictorsetmaker 328
+#define classdef 329
+#define arglist 330
+#define argument 331
+#define list_iter 332
+#define list_for 333
+#define list_if 334
+#define comp_iter 335
+#define comp_for 336
+#define comp_if 337
+#define testlist1 338
+#define encoding_decl 339
+#define yield_expr 340

from_cpython/Include/grammar.h

+
+/* Grammar interface */
+
+#ifndef Py_GRAMMAR_H
+#define Py_GRAMMAR_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "bitset.h" /* Sigh... */
+
+/* A label of an arc */
+
+typedef struct {
+    int		 lb_type;
+    char	*lb_str;
+} label;
+
+#define EMPTY 0		/* Label number 0 is by definition the empty label */
+
+/* A list of labels */
+
+typedef struct {
+    int		 ll_nlabels;
+    label	*ll_label;
+} labellist;
+
+/* An arc from one state to another */
+
+typedef struct {
+    short	a_lbl;		/* Label of this arc */
+    short	a_arrow;	/* State where this arc goes to */
+} arc;
+
+/* A state in a DFA */
+
+typedef struct {
+    int		 s_narcs;
+    arc		*s_arc;		/* Array of arcs */
+	
+    /* Optional accelerators */
+    int		 s_lower;	/* Lowest label index */
+    int		 s_upper;	/* Highest label index */
+    int		*s_accel;	/* Accelerator */
+    int		 s_accept;	/* Nonzero for accepting state */
+} state;
+
+/* A DFA */
+
+typedef struct {
+    int		 d_type;	/* Non-terminal this represents */
+    char	*d_name;	/* For printing */
+    int		 d_initial;	/* Initial state */
+    int		 d_nstates;
+    state	*d_state;	/* Array of states */
+    bitset	 d_first;
+} dfa;
+
+/* A grammar */
+
+typedef struct {
+    int		 g_ndfas;
+    dfa		*g_dfa;		/* Array of DFAs */
+    labellist	 g_ll;
+    int		 g_start;	/* Start symbol of the grammar */
+    int		 g_accel;	/* Set if accelerators present */
+} grammar;
+
+/* FUNCTIONS */
+
+grammar *newgrammar(int start);
+dfa *adddfa(grammar *g, int type, char *name);
+int addstate(dfa *d);
+void addarc(dfa *d, int from, int to, int lbl);
+dfa *PyGrammar_FindDFA(grammar *g, int type);
+
+int addlabel(labellist *ll, int type, char *str);
+int findlabel(labellist *ll, int type, char *str);
+char *PyGrammar_LabelRepr(label *lb);
+void translatelabels(grammar *g);
+
+void addfirstsets(grammar *g);
+
+void PyGrammar_AddAccelerators(grammar *g);
+void PyGrammar_RemoveAccelerators(grammar *);
+
+void printgrammar(grammar *g, FILE *fp);
+void printnonterminals(grammar *g, FILE *fp);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GRAMMAR_H */

from_cpython/Include/node.h

+
+/* Parse tree node interface */
+
+#ifndef Py_NODE_H
+#define Py_NODE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct _node {
+    short		n_type;
+    char		*n_str;
+    int			n_lineno;
+    int			n_col_offset;
+    int			n_nchildren;
+    struct _node	*n_child;
+} node;
+
+PyAPI_FUNC(node *) PyNode_New(int type);
+PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
+                                      char *str, int lineno, int col_offset);
+PyAPI_FUNC(void) PyNode_Free(node *n);
+#ifndef Py_LIMITED_API
+Py_ssize_t _PyNode_SizeOf(node *n);
+#endif
+
+/* Node access functions */
+#define NCH(n)		((n)->n_nchildren)
+	
+#define CHILD(n, i)	(&(n)->n_child[i])
+#define RCHILD(n, i)	(CHILD(n, NCH(n) + i))
+#define TYPE(n)		((n)->n_type)
+#define STR(n)		((n)->n_str)
+
+/* Assert that the type of a node is what we expect */
+#define REQ(n, type) assert(TYPE(n) == (type))
+
+PyAPI_FUNC(void) PyNode_ListTree(node *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_NODE_H */

from_cpython/Include/object.h

@@ -459,7 +459,7 @@ struct _typeobject {
     char _ics[32];
     void* _gcvisit_func;
     int _attrs_offset;
-    bool _flags[7];
+    char _flags[7]; // These are bools in C++
     void* _tpp_descr_get;
     void* _tpp_hasnext;
     void* _tpp_call_capi;

from_cpython/Include/parsetok.h

+
+/* Parser-tokenizer link interface */
+
+#ifndef Py_PARSETOK_H
+#define Py_PARSETOK_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    int error;
+    const char *filename;
+    int lineno;
+    int offset;
+    char *text;
+    int token;
+    int expected;
+} perrdetail;
+
+#if 0
+#define PyPARSE_YIELD_IS_KEYWORD	0x0001
+#endif
+
+#define PyPARSE_DONT_IMPLY_DEDENT	0x0002
+
+#if 0
+#define PyPARSE_WITH_IS_KEYWORD		0x0003
+#endif
+
+#define PyPARSE_PRINT_IS_FUNCTION       0x0004
+#define PyPARSE_UNICODE_LITERALS        0x0008
+
+
+
+PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
+                                              perrdetail *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
+                                             char *, char *, perrdetail *) PYSTON_NOEXCEPT;
+
+PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
+                                              perrdetail *, int) PYSTON_NOEXCEPT;
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
+						 int, char *, char *,
+						 perrdetail *, int) PYSTON_NOEXCEPT;
+PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *, grammar *,
+						 int, char *, char *,
+						 perrdetail *, int *) PYSTON_NOEXCEPT;
+
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
+					      const char *,
+					      grammar *, int,
+                                              perrdetail *, int) PYSTON_NOEXCEPT;
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
+					      const char *,
+					      grammar *, int,
+                                              perrdetail *, int *) PYSTON_NOEXCEPT;
+
+/* Note that he following function is defined in pythonrun.c not parsetok.c. */
+PyAPI_FUNC(void) PyParser_SetError(perrdetail *) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PARSETOK_H */

from_cpython/Include/pgenheaders.h

+#ifndef Py_PGENHEADERS_H
+#define Py_PGENHEADERS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Include files and extern declarations used by most of the parser. */
+
+#include "Python.h"
+
+PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...)
+			Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
+PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...)
+			Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
+
+#define addarc _Py_addarc
+#define addbit _Py_addbit
+#define adddfa _Py_adddfa
+#define addfirstsets _Py_addfirstsets
+#define addlabel _Py_addlabel
+#define addstate _Py_addstate
+#define delbitset _Py_delbitset
+#define dumptree _Py_dumptree
+#define findlabel _Py_findlabel
+#define mergebitset _Py_mergebitset
+#define meta_grammar _Py_meta_grammar
+#define newbitset _Py_newbitset
+#define newgrammar _Py_newgrammar
+#define pgen _Py_pgen
+#define printgrammar _Py_printgrammar
+#define printnonterminals _Py_printnonterminals
+#define printtree _Py_printtree
+#define samebitset _Py_samebitset
+#define showtree _Py_showtree
+#define tok_dump _Py_tok_dump
+#define translatelabels _Py_translatelabels
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PGENHEADERS_H */

from_cpython/Include/pystrcmp.h

+#ifndef Py_STRCMP_H
+#define Py_STRCMP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(int) PyOS_mystrnicmp(const char *, const char *, Py_ssize_t);
+PyAPI_FUNC(int) PyOS_mystricmp(const char *, const char *);
+
+#if defined(MS_WINDOWS) || defined(PYOS_OS2)
+#define PyOS_strnicmp strnicmp
+#define PyOS_stricmp stricmp
+#else
+#define PyOS_strnicmp PyOS_mystrnicmp
+#define PyOS_stricmp PyOS_mystricmp
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_STRCMP_H */

from_cpython/Include/symtable.h

+#ifndef Py_SYMTABLE_H
+#define Py_SYMTABLE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
+    _Py_block_ty;
+
+struct _symtable_entry;
+
+struct symtable {
+    const char *st_filename; /* name of file being compiled */
+    struct _symtable_entry *st_cur; /* current symbol table entry */
+    struct _symtable_entry *st_top; /* module entry */
+    PyObject *st_symbols;    /* dictionary of symbol table entries */
+    PyObject *st_stack;      /* stack of namespace info */
+    PyObject *st_global;     /* borrowed ref to MODULE in st_symbols */
+    int st_nblocks;          /* number of blocks */
+    PyObject *st_private;        /* name of current class or NULL */
+    PyFutureFeatures *st_future; /* module's future features */
+};
+
+typedef struct _symtable_entry {
+    PyObject_HEAD
+    PyObject *ste_id;        /* int: key in st_symbols */
+    PyObject *ste_symbols;   /* dict: name to flags */
+    PyObject *ste_name;      /* string: name of block */
+    PyObject *ste_varnames;  /* list of variable names */
+    PyObject *ste_children;  /* list of child ids */
+    _Py_block_ty ste_type;   /* module, class, or function */
+    int ste_unoptimized;     /* false if namespace is optimized */
+    int ste_nested;      /* true if block is nested */
+    unsigned ste_free : 1;        /* true if block has free variables */
+    unsigned ste_child_free : 1;  /* true if a child block has free vars,
+                                     including free refs to globals */
+    unsigned ste_generator : 1;   /* true if namespace is a generator */
+    unsigned ste_varargs : 1;     /* true if block has varargs */
+    unsigned ste_varkeywords : 1; /* true if block has varkeywords */
+    unsigned ste_returns_value : 1;  /* true if namespace uses return with
+                                        an argument */
+    int ste_lineno;          /* first line of block */
+    int ste_opt_lineno;      /* lineno of last exec or import * */
+    int ste_tmpname;         /* counter for listcomp temp vars */
+    struct symtable *ste_table;
+} PySTEntryObject;
+
+PyAPI_DATA(PyTypeObject) PySTEntry_Type;
+
+#define PySTEntry_Check(op) (Py_TYPE(op) == &PySTEntry_Type)
+
+PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
+
+PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
+                                              PyFutureFeatures *);
+PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
+
+PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
+
+/* Flags for def-use information */
+
+#define DEF_GLOBAL 1           /* global stmt */
+#define DEF_LOCAL 2            /* assignment in code block */
+#define DEF_PARAM 2<<1         /* formal parameter */
+#define USE 2<<2               /* name is used */
+#define DEF_FREE 2<<3         /* name used but not defined in nested block */
+#define DEF_FREE_CLASS 2<<4    /* free variable from class's method */
+#define DEF_IMPORT 2<<5        /* assignment occurred via import */
+
+#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
+
+/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
+   table.  GLOBAL is returned from PyST_GetScope() for either of them.
+   It is stored in ste_symbols at bits 12-14.
+*/
+#define SCOPE_OFF 11
+#define SCOPE_MASK 7
+
+#define LOCAL 1
+#define GLOBAL_EXPLICIT 2
+#define GLOBAL_IMPLICIT 3
+#define FREE 4
+#define CELL 5
+
+/* The following three names are used for the ste_unoptimized bit field */
+#define OPT_IMPORT_STAR 1
+#define OPT_EXEC 2
+#define OPT_BARE_EXEC 4
+#define OPT_TOPLEVEL 8  /* top-level names, including eval and exec */
+
+#define GENERATOR 1
+#define GENERATOR_EXPRESSION 2
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SYMTABLE_H */

from_cpython/Include/token.h

+
+/* Token types */
+
+#ifndef Py_TOKEN_H
+#define Py_TOKEN_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#undef TILDE   /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
+
+#define ENDMARKER	0
+#define NAME		1
+#define NUMBER		2
+#define STRING		3
+#define NEWLINE		4
+#define INDENT		5
+#define DEDENT		6
+#define LPAR		7
+#define RPAR		8
+#define LSQB		9
+#define RSQB		10
+#define COLON		11
+#define COMMA		12
+#define SEMI		13
+#define PLUS		14
+#define MINUS		15
+#define STAR		16
+#define SLASH		17
+#define VBAR		18
+#define AMPER		19
+#define LESS		20
+#define GREATER		21
+#define EQUAL		22
+#define DOT		23
+#define PERCENT		24
+#define BACKQUOTE	25
+#define LBRACE		26
+#define RBRACE		27
+#define EQEQUAL		28
+#define NOTEQUAL	29
+#define LESSEQUAL	30
+#define GREATEREQUAL	31
+#define TILDE		32
+#define CIRCUMFLEX	33
+#define LEFTSHIFT	34
+#define RIGHTSHIFT	35
+#define DOUBLESTAR	36
+#define PLUSEQUAL	37
+#define MINEQUAL	38
+#define STAREQUAL	39
+#define SLASHEQUAL	40
+#define PERCENTEQUAL	41
+#define AMPEREQUAL	42
+#define VBAREQUAL	43
+#define CIRCUMFLEXEQUAL	44
+#define LEFTSHIFTEQUAL	45
+#define RIGHTSHIFTEQUAL	46
+#define DOUBLESTAREQUAL	47
+#define DOUBLESLASH	48
+#define DOUBLESLASHEQUAL 49
+#define AT              50	
+/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
+#define OP		51
+#define ERRORTOKEN	52
+#define N_TOKENS	53
+
+/* Special definitions for cooperation with parser */
+
+#define NT_OFFSET		256
+
+#define ISTERMINAL(x)		((x) < NT_OFFSET)
+#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
+#define ISEOF(x)		((x) == ENDMARKER)
+
+
+PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */
+PyAPI_FUNC(int) PyToken_OneChar(int);
+PyAPI_FUNC(int) PyToken_TwoChars(int, int);
+PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_TOKEN_H */

from_cpython/Modules/_ctypes/_ctypes.c

@@ -427,7 +427,7 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)dict;
-    PyType_SetDict(result, dict);
+    PyType_SetDict(result, (PyObject*)dict);
     dict->format = _ctypes_alloc_format_string(NULL, "B");
     if (dict->format == NULL) {
         Py_DECREF(result);
@@ -999,7 +999,7 @@ PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)stgdict;
-    PyType_SetDict(result, stgdict);
+    PyType_SetDict(result, (PyObject*)stgdict);
 
     return (PyObject *)result;
 }
@@ -1467,7 +1467,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)stgdict;
-    PyType_SetDict(result, stgdict);
+    PyType_SetDict(result, (PyObject*)stgdict);
 
     /* Special case for character arrays.
        A permanent annoyance: char arrays are also strings!
@@ -1893,7 +1893,7 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)stgdict;
-    PyType_SetDict(result, stgdict);
+    PyType_SetDict(result, (PyObject*)stgdict);
 
     return (PyObject *)result;
 }
@@ -2024,7 +2024,7 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)stgdict;
-    PyType_SetDict(result, stgdict);
+    PyType_SetDict(result, (PyObject*)stgdict);
 
     /* Install from_param class methods in ctypes base classes.
        Overrides the PyCSimpleType_from_param generic method.
@@ -2407,7 +2407,7 @@ PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     Py_DECREF(result->tp_dict);
     // Pyston change:
     //result->tp_dict = (PyObject *)stgdict;
-    PyType_SetDict(result, stgdict);
+    PyType_SetDict(result, (PyObject*)stgdict);
 
     if (-1 == make_funcptrtype_dict(stgdict)) {
         Py_DECREF(result);

from_cpython/Modules/_ctypes/cfield.c

@@ -1293,7 +1293,7 @@ s_get(void *ptr, Py_ssize_t size)
     size = min(size, (Py_ssize_t)slen);
 
     // Pyston change: no ob_refcnt
-    if (false /*result->ob_refcnt == 1*/) {
+    if (0 /*result->ob_refcnt == 1*/) {
         /* shorten the result */
         _PyString_Resize(&result, size);
         return result;

from_cpython/Objects/weakrefobject.c

@@ -951,7 +951,7 @@ PyObject_ClearWeakRefs(PyObject *object)
                 // Pyston change:
                 // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
                 // be collected, and we can trust that it's still valid here.
-                if (true /*current->ob_refcnt > 0*/)
+                if (1 /*current->ob_refcnt > 0*/)
                     handle_callback(current, callback);
                 Py_DECREF(callback);
             }
@@ -973,7 +973,7 @@ PyObject_ClearWeakRefs(PyObject *object)
                 // Pyston change:
                 // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
                 // be collected, and we can trust that it's still valid here.
-                if (true /*current->ob_refcnt > 0*/)
+                if (1 /*current->ob_refcnt > 0*/)
                 {
                     Py_INCREF(current);
                     PyTuple_SET_ITEM(tuple, i * 2, (PyObject *) current);

from_cpython/Parser/Python.asdl

+-- ASDL's five builtin types are identifier, int, string, object, bool
+
+module Python version "$Revision$"
+{
+	mod = Module(stmt* body)
+	    | Interactive(stmt* body)
+	    | Expression(expr body)
+
+	    -- not really an actual node but useful in Jython's typesystem.
+	    | Suite(stmt* body)
+
+	stmt = FunctionDef(identifier name, arguments args, 
+                            stmt* body, expr* decorator_list)
+	      | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
+	      | Return(expr? value)
+
+	      | Delete(expr* targets)
+	      | Assign(expr* targets, expr value)
+	      | AugAssign(expr target, operator op, expr value)
+
+	      -- not sure if bool is allowed, can always use int
+ 	      | Print(expr? dest, expr* values, bool nl)
+
+	      -- use 'orelse' because else is a keyword in target languages
+	      | For(expr target, expr iter, stmt* body, stmt* orelse)
+	      | While(expr test, stmt* body, stmt* orelse)
+	      | If(expr test, stmt* body, stmt* orelse)
+	      | With(expr context_expr, expr? optional_vars, stmt* body)
+
+	      -- 'type' is a bad name
+	      | Raise(expr? type, expr? inst, expr? tback)
+	      | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
+	      | TryFinally(stmt* body, stmt* finalbody)
+	      | Assert(expr test, expr? msg)
+
+	      | Import(alias* names)
+	      | ImportFrom(identifier? module, alias* names, int? level)
+
+	      -- Doesn't capture requirement that locals must be
+	      -- defined if globals is
+	      -- still supports use as a function!
+	      | Exec(expr body, expr? globals, expr? locals)
+
+	      | Global(identifier* names)
+	      | Expr(expr value)
+	      | Pass | Break | Continue
+
+	      -- XXX Jython will be different
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
+
+	      -- BoolOp() can use left & right?
+	expr = BoolOp(boolop op, expr* values)
+	     | BinOp(expr left, operator op, expr right)
+	     | UnaryOp(unaryop op, expr operand)
+	     | Lambda(arguments args, expr body)
+	     | IfExp(expr test, expr body, expr orelse)
+	     | Dict(expr* keys, expr* values)
+	     | Set(expr* elts)
+	     | ListComp(expr elt, comprehension* generators)
+	     | SetComp(expr elt, comprehension* generators)
+	     | DictComp(expr key, expr value, comprehension* generators)
+	     | GeneratorExp(expr elt, comprehension* generators)
+	     -- the grammar constrains where yield expressions can occur
+	     | Yield(expr? value)
+	     -- need sequences for compare to distinguish between
+	     -- x < 4 < 3 and (x < 4) < 3
+	     | Compare(expr left, cmpop* ops, expr* comparators)
+	     | Call(expr func, expr* args, keyword* keywords,
+			 expr? starargs, expr? kwargs)
+	     | Repr(expr value)
+	     | Num(object n) -- a number as a PyObject.
+	     | Str(string s) -- need to specify raw, unicode, etc?
+	     -- other literals? bools?
+
+	     -- the following expression can appear in assignment context
+	     | Attribute(expr value, identifier attr, expr_context ctx)
+	     | Subscript(expr value, slice slice, expr_context ctx)
+	     | Name(identifier id, expr_context ctx)
+	     | List(expr* elts, expr_context ctx) 
+	     | Tuple(expr* elts, expr_context ctx)
+
+	      -- col_offset is the byte offset in the utf8 string the parser uses
+	      attributes (int lineno, int col_offset)
+
+	expr_context = Load | Store | Del | AugLoad | AugStore | Param
+
+	slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) 
+	      | ExtSlice(slice* dims) 
+	      | Index(expr value) 
+
+	boolop = And | Or 
+
+	operator = Add | Sub | Mult | Div | Mod | Pow | LShift 
+                 | RShift | BitOr | BitXor | BitAnd | FloorDiv
+
+	unaryop = Invert | Not | UAdd | USub
+
+	cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
+
+	comprehension = (expr target, expr iter, expr* ifs)
+
+	-- not sure what to call the first argument for raise and except
+	excepthandler = ExceptHandler(expr? type, expr? name, stmt* body)
+	                attributes (int lineno, int col_offset)
+
+	arguments = (expr* args, identifier? vararg, 
+		     identifier? kwarg, expr* defaults)
+
+        -- keyword arguments supplied to call
+        keyword = (identifier arg, expr value)
+
+        -- import name with optional 'as' alias.
+        alias = (identifier name, identifier? asname)
+}

from_cpython/Parser/acceler.c

+
+/* Parser accelerator module */
+
+/* The parser as originally conceived had disappointing performance.
+   This module does some precomputation that speeds up the selection
+   of a DFA based upon a token, turning a search through an array
+   into a simple indexing operation.  The parser now cannot work
+   without the accelerators installed.  Note that the accelerators
+   are installed dynamically when the parser is initialized, they
+   are not part of the static data structure written on graminit.[ch]
+   by the parser generator. */
+
+#include "pgenheaders.h"
+#include "grammar.h"
+#include "node.h"
+#include "token.h"
+#include "parser.h"
+
+/* Forward references */
+static void fixdfa(grammar *, dfa *);
+static void fixstate(grammar *, state *);
+
+void
+PyGrammar_AddAccelerators(grammar *g)
+{
+    dfa *d;
+    int i;
+    d = g->g_dfa;
+    for (i = g->g_ndfas; --i >= 0; d++)
+        fixdfa(g, d);
+    g->g_accel = 1;
+}
+
+void
+PyGrammar_RemoveAccelerators(grammar *g)
+{
+    dfa *d;
+    int i;
+    g->g_accel = 0;
+    d = g->g_dfa;
+    for (i = g->g_ndfas; --i >= 0; d++) {
+        state *s;
+        int j;
+        s = d->d_state;
+        for (j = 0; j < d->d_nstates; j++, s++) {
+            if (s->s_accel)
+                PyObject_FREE(s->s_accel);
+            s->s_accel = NULL;
+        }
+    }
+}
+
+static void
+fixdfa(grammar *g, dfa *d)
+{
+    state *s;
+    int j;
+    s = d->d_state;
+    for (j = 0; j < d->d_nstates; j++, s++)
+        fixstate(g, s);
+}
+
+static void
+fixstate(grammar *g, state *s)
+{
+    arc *a;
+    int k;
+    int *accel;
+    int nl = g->g_ll.ll_nlabels;
+    s->s_accept = 0;
+    accel = (int *) PyObject_MALLOC(nl * sizeof(int));
+    if (accel == NULL) {
+        fprintf(stderr, "no mem to build parser accelerators\n");
+        exit(1);
+    }
+    for (k = 0; k < nl; k++)
+        accel[k] = -1;
+    a = s->s_arc;
+    for (k = s->s_narcs; --k >= 0; a++) {
+        int lbl = a->a_lbl;
+        label *l = &g->g_ll.ll_label[lbl];
+        int type = l->lb_type;
+        if (a->a_arrow >= (1 << 7)) {
+            printf("XXX too many states!\n");
+            continue;
+        }
+        if (ISNONTERMINAL(type)) {
+            dfa *d1 = PyGrammar_FindDFA(g, type);
+            int ibit;
+            if (type - NT_OFFSET >= (1 << 7)) {
+                printf("XXX too high nonterminal number!\n");
+                continue;
+            }
+            for (ibit = 0; ibit < g->g_ll.ll_nlabels; ibit++) {
+                if (testbit(d1->d_first, ibit)) {
+                    if (accel[ibit] != -1)
+                        printf("XXX ambiguity!\n");
+                    accel[ibit] = a->a_arrow | (1 << 7) |
+                        ((type - NT_OFFSET) << 8);
+                }
+            }
+        }
+        else if (lbl == EMPTY)
+            s->s_accept = 1;
+        else if (lbl >= 0 && lbl < nl)
+            accel[lbl] = a->a_arrow;
+    }
+    while (nl > 0 && accel[nl-1] == -1)
+        nl--;
+    for (k = 0; k < nl && accel[k] == -1;)
+        k++;
+    if (k < nl) {
+        int i;
+        // Pyston change:
+        s->s_accel = (int *) PyGC_AddRoot(PyObject_MALLOC((nl-k) * sizeof(int)));
+        if (s->s_accel == NULL) {
+            fprintf(stderr, "no mem to add parser accelerators\n");
+            exit(1);
+        }
+        s->s_lower = k;
+        s->s_upper = nl;
+        for (i = 0; k < nl; i++, k++)
+            s->s_accel[i] = accel[k];
+    }
+    PyObject_FREE(accel);
+}

from_cpython/Parser/bitset.c

+
+/* Bitset primitives used by the parser generator */
+
+#include "pgenheaders.h"
+#include "bitset.h"
+
+bitset
+newbitset(int nbits)
+{
+    int nbytes = NBYTES(nbits);
+    bitset ss = (char *)PyObject_MALLOC(sizeof(BYTE) *  nbytes);
+
+    if (ss == NULL)
+        Py_FatalError("no mem for bitset");
+
+    ss += nbytes;
+    while (--nbytes >= 0)
+        *--ss = 0;
+    return ss;
+}
+
+void
+delbitset(bitset ss)
+{
+    PyObject_FREE(ss);
+}
+
+int
+addbit(bitset ss, int ibit)
+{
+    int ibyte = BIT2BYTE(ibit);
+    BYTE mask = BIT2MASK(ibit);
+
+    if (ss[ibyte] & mask)
+        return 0; /* Bit already set */
+    ss[ibyte] |= mask;
+    return 1;
+}
+
+#if 0 /* Now a macro */
+int
+testbit(bitset ss, int ibit)
+{
+    return (ss[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0;
+}
+#endif
+
+int
+samebitset(bitset ss1, bitset ss2, int nbits)
+{
+    int i;
+
+    for (i = NBYTES(nbits); --i >= 0; )
+        if (*ss1++ != *ss2++)
+            return 0;
+    return 1;
+}
+
+void
+mergebitset(bitset ss1, bitset ss2, int nbits)
+{
+    int i;
+
+    for (i = NBYTES(nbits); --i >= 0; )
+        *ss1++ |= *ss2++;
+}

from_cpython/Parser/firstsets.c

+
+/* Computation of FIRST stets */
+
+#include "pgenheaders.h"
+#include "grammar.h"
+#include "token.h"
+
+extern int Py_DebugFlag;
+
+/* Forward */
+static void calcfirstset(grammar *, dfa *);
+
+void
+addfirstsets(grammar *g)
+{
+    int i;
+    dfa *d;
+
+    if (Py_DebugFlag)
+        printf("Adding FIRST sets ...\n");
+    for (i = 0; i < g->g_ndfas; i++) {
+        d = &g->g_dfa[i];
+        if (d->d_first == NULL)
+            calcfirstset(g, d);
+    }
+}
+
+static void
+calcfirstset(grammar *g, dfa *d)
+{
+    int i, j;
+    state *s;
+    arc *a;
+    int nsyms;
+    int *sym;
+    int nbits;
+    static bitset dummy;
+    bitset result;
+    int type;
+    dfa *d1;
+    label *l0;
+
+    if (Py_DebugFlag)
+        printf("Calculate FIRST set for '%s'\n", d->d_name);
+
+    if (dummy == NULL)
+        dummy = newbitset(1);
+    if (d->d_first == dummy) {
+        fprintf(stderr, "Left-recursion for '%s'\n", d->d_name);
+        return;
+    }
+    if (d->d_first != NULL) {
+        fprintf(stderr, "Re-calculating FIRST set for '%s' ???\n",
+            d->d_name);
+    }
+    d->d_first = dummy;
+
+    l0 = g->g_ll.ll_label;
+    nbits = g->g_ll.ll_nlabels;
+    result = newbitset(nbits);
+
+    sym = (int *)PyObject_MALLOC(sizeof(int));
+    if (sym == NULL)
+        Py_FatalError("no mem for new sym in calcfirstset");
+    nsyms = 1;
+    sym[0] = findlabel(&g->g_ll, d->d_type, (char *)NULL);
+
+    s = &d->d_state[d->d_initial];
+    for (i = 0; i < s->s_narcs; i++) {
+        a = &s->s_arc[i];
+        for (j = 0; j < nsyms; j++) {
+            if (sym[j] == a->a_lbl)
+                break;
+        }
+        if (j >= nsyms) { /* New label */
+            sym = (int *)PyObject_REALLOC(sym,
+                                    sizeof(int) * (nsyms + 1));
+            if (sym == NULL)
+                Py_FatalError(
+                    "no mem to resize sym in calcfirstset");
+            sym[nsyms++] = a->a_lbl;
+            type = l0[a->a_lbl].lb_type;
+            if (ISNONTERMINAL(type)) {
+                d1 = PyGrammar_FindDFA(g, type);
+                if (d1->d_first == dummy) {
+                    fprintf(stderr,
+                        "Left-recursion below '%s'\n",
+                        d->d_name);
+                }
+                else {
+                    if (d1->d_first == NULL)
+                        calcfirstset(g, d1);
+                    mergebitset(result,
+                                d1->d_first, nbits);
+                }
+            }
+            else if (ISTERMINAL(type)) {
+                addbit(result, a->a_lbl);
+            }
+        }
+    }
+    d->d_first = result;
+    if (Py_DebugFlag) {
+        printf("FIRST set for '%s': {", d->d_name);
+        for (i = 0; i < nbits; i++) {
+            if (testbit(result, i))
+                printf(" %s", PyGrammar_LabelRepr(&l0[i]));
+        }
+        printf(" }\n");
+    }
+
+    PyObject_FREE(sym);
+}

from_cpython/Parser/grammar.c

+
+/* Grammar implementation */
+
+#include "Python.h"
+#include "pgenheaders.h"
+
+#include <ctype.h>
+
+#include "token.h"
+#include "grammar.h"
+
+#ifdef RISCOS
+#include <unixlib.h>
+#endif
+
+extern int Py_DebugFlag;
+
+grammar *
+newgrammar(int start)
+{
+    grammar *g;
+
+    g = (grammar *)PyObject_MALLOC(sizeof(grammar));
+    if (g == NULL)
+        Py_FatalError("no mem for new grammar");
+    g->g_ndfas = 0;
+    g->g_dfa = NULL;
+    g->g_start = start;
+    g->g_ll.ll_nlabels = 0;
+    g->g_ll.ll_label = NULL;
+    g->g_accel = 0;
+    return g;
+}
+
+dfa *
+adddfa(grammar *g, int type, char *name)
+{
+    dfa *d;
+
+    g->g_dfa = (dfa *)PyObject_REALLOC(g->g_dfa,
+                                        sizeof(dfa) * (g->g_ndfas + 1));
+    if (g->g_dfa == NULL)
+        Py_FatalError("no mem to resize dfa in adddfa");
+    d = &g->g_dfa[g->g_ndfas++];
+    d->d_type = type;
+    d->d_name = strdup(name);
+    d->d_nstates = 0;
+    d->d_state = NULL;
+    d->d_initial = -1;
+    d->d_first = NULL;
+    return d; /* Only use while fresh! */
+}
+
+int
+addstate(dfa *d)
+{
+    state *s;
+
+    d->d_state = (state *)PyObject_REALLOC(d->d_state,
+                                  sizeof(state) * (d->d_nstates + 1));
+    if (d->d_state == NULL)
+        Py_FatalError("no mem to resize state in addstate");
+    s = &d->d_state[d->d_nstates++];
+    s->s_narcs = 0;
+    s->s_arc = NULL;
+    s->s_lower = 0;
+    s->s_upper = 0;
+    s->s_accel = NULL;
+    s->s_accept = 0;
+    return s - d->d_state;
+}
+
+void
+addarc(dfa *d, int from, int to, int lbl)
+{
+    state *s;
+    arc *a;
+
+    assert(0 <= from && from < d->d_nstates);
+    assert(0 <= to && to < d->d_nstates);
+
+    s = &d->d_state[from];
+    s->s_arc = (arc *)PyObject_REALLOC(s->s_arc, sizeof(arc) * (s->s_narcs + 1));
+    if (s->s_arc == NULL)
+        Py_FatalError("no mem to resize arc list in addarc");
+    a = &s->s_arc[s->s_narcs++];
+    a->a_lbl = lbl;
+    a->a_arrow = to;
+}
+
+int
+addlabel(labellist *ll, int type, char *str)
+{
+    int i;
+    label *lb;
+
+    for (i = 0; i < ll->ll_nlabels; i++) {
+        if (ll->ll_label[i].lb_type == type &&
+            strcmp(ll->ll_label[i].lb_str, str) == 0)
+            return i;
+    }
+    ll->ll_label = (label *)PyObject_REALLOC(ll->ll_label,
+                                    sizeof(label) * (ll->ll_nlabels + 1));
+    if (ll->ll_label == NULL)
+        Py_FatalError("no mem to resize labellist in addlabel");
+    lb = &ll->ll_label[ll->ll_nlabels++];
+    lb->lb_type = type;
+    lb->lb_str = strdup(str);
+    if (Py_DebugFlag)
+        printf("Label @ %8p, %d: %s\n", ll, ll->ll_nlabels,
+               PyGrammar_LabelRepr(lb));
+    return lb - ll->ll_label;
+}
+
+/* Same, but rather dies than adds */
+
+int
+findlabel(labellist *ll, int type, char *str)
+{
+    int i;
+
+    for (i = 0; i < ll->ll_nlabels; i++) {
+        if (ll->ll_label[i].lb_type == type /*&&
+            strcmp(ll->ll_label[i].lb_str, str) == 0*/)
+            return i;
+    }
+    fprintf(stderr, "Label %d/'%s' not found\n", type, str);
+    Py_FatalError("grammar.c:findlabel()");
+    return 0; /* Make gcc -Wall happy */
+}
+
+/* Forward */
+static void translabel(grammar *, label *);
+
+void
+translatelabels(grammar *g)
+{
+    int i;
+
+#ifdef Py_DEBUG
+    printf("Translating labels ...\n");
+#endif
+    /* Don't translate EMPTY */
+    for (i = EMPTY+1; i < g->g_ll.ll_nlabels; i++)
+        translabel(g, &g->g_ll.ll_label[i]);
+}
+
+static void
+translabel(grammar *g, label *lb)
+{
+    int i;
+
+    if (Py_DebugFlag)
+        printf("Translating label %s ...\n", PyGrammar_LabelRepr(lb));
+
+    if (lb->lb_type == NAME) {
+        for (i = 0; i < g->g_ndfas; i++) {
+            if (strcmp(lb->lb_str, g->g_dfa[i].d_name) == 0) {
+                if (Py_DebugFlag)
+                    printf(
+                        "Label %s is non-terminal %d.\n",
+                        lb->lb_str,
+                        g->g_dfa[i].d_type);
+                lb->lb_type = g->g_dfa[i].d_type;
+                free(lb->lb_str);
+                lb->lb_str = NULL;
+                return;
+            }
+        }
+        for (i = 0; i < (int)N_TOKENS; i++) {
+            if (strcmp(lb->lb_str, _PyParser_TokenNames[i]) == 0) {
+                if (Py_DebugFlag)
+                    printf("Label %s is terminal %d.\n",
+                        lb->lb_str, i);
+                lb->lb_type = i;
+                free(lb->lb_str);
+                lb->lb_str = NULL;
+                return;
+            }
+        }
+        printf("Can't translate NAME label '%s'\n", lb->lb_str);
+        return;
+    }
+
+    if (lb->lb_type == STRING) {
+        if (isalpha(Py_CHARMASK(lb->lb_str[1])) ||
+            lb->lb_str[1] == '_') {
+            char *p;
+            char *src;
+            char *dest;
+            size_t name_len;
+            if (Py_DebugFlag)
+                printf("Label %s is a keyword\n", lb->lb_str);
+            lb->lb_type = NAME;
+            src = lb->lb_str + 1;
+            p = strchr(src, '\'');
+            if (p)
+                name_len = p - src;
+            else
+                name_len = strlen(src);
+            dest = (char *)malloc(name_len + 1);
+            if (!dest) {
+                printf("Can't alloc dest '%s'\n", src);
+                return;
+            }
+            strncpy(dest, src, name_len);
+            dest[name_len] = '\0';
+            free(lb->lb_str);
+            lb->lb_str = dest;
+        }
+        else if (lb->lb_str[2] == lb->lb_str[0]) {
+            int type = (int) PyToken_OneChar(lb->lb_str[1]);
+            if (type != OP) {
+                lb->lb_type = type;
+                free(lb->lb_str);
+                lb->lb_str = NULL;
+            }
+            else
+                printf("Unknown OP label %s\n",
+                    lb->lb_str);
+        }
+        else if (lb->lb_str[2] && lb->lb_str[3] == lb->lb_str[0]) {
+            int type = (int) PyToken_TwoChars(lb->lb_str[1],
+                                       lb->lb_str[2]);
+            if (type != OP) {
+                lb->lb_type = type;
+                free(lb->lb_str);
+                lb->lb_str = NULL;
+            }
+            else
+                printf("Unknown OP label %s\n",
+                    lb->lb_str);
+        }
+        else if (lb->lb_str[2] && lb->lb_str[3] && lb->lb_str[4] == lb->lb_str[0]) {
+            int type = (int) PyToken_ThreeChars(lb->lb_str[1],
+                                                lb->lb_str[2],
+                                                lb->lb_str[3]);
+            if (type != OP) {
+                lb->lb_type = type;
+                free(lb->lb_str);
+                lb->lb_str = NULL;
+            }
+            else
+                printf("Unknown OP label %s\n",
+                    lb->lb_str);
+        }
+        else
+            printf("Can't translate STRING label %s\n",
+                lb->lb_str);
+    }
+    else
+        printf("Can't translate label '%s'\n",
+               PyGrammar_LabelRepr(lb));
+}

from_cpython/Parser/grammar1.c

+
+/* Grammar subroutines needed by parser */
+
+#include "Python.h"
+#include "pgenheaders.h"
+#include "grammar.h"
+#include "token.h"
+
+/* Return the DFA for the given type */
+
+dfa *
+PyGrammar_FindDFA(grammar *g, register int type)
+{
+    register dfa *d;
+#if 1
+    /* Massive speed-up */
+    d = &g->g_dfa[type - NT_OFFSET];
+    assert(d->d_type == type);
+    return d;
+#else
+    /* Old, slow version */
+    register int i;
+
+    for (i = g->g_ndfas, d = g->g_dfa; --i >= 0; d++) {
+        if (d->d_type == type)
+            return d;
+    }
+    assert(0);
+    /* NOTREACHED */
+#endif
+}
+
+char *
+PyGrammar_LabelRepr(label *lb)
+{
+    static char buf[100];
+
+    if (lb->lb_type == ENDMARKER)
+        return "EMPTY";
+    else if (ISNONTERMINAL(lb->lb_type)) {
+        if (lb->lb_str == NULL) {
+            PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type);
+            return buf;
+        }
+        else
+            return lb->lb_str;
+    }
+    else {
+        if (lb->lb_str == NULL)
+            return _PyParser_TokenNames[lb->lb_type];
+        else {
+            PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)",
+                _PyParser_TokenNames[lb->lb_type], lb->lb_str);
+            return buf;
+        }
+    }
+}

from_cpython/Parser/intrcheck.c

+
+/* Check for interrupts */
+
+#include "Python.h"
+#include "pythread.h"
+
+#ifdef QUICKWIN
+
+#include <io.h>
+
+void
+PyOS_InitInterrupts(void)
+{
+}
+
+void
+PyOS_FiniInterrupts(void)
+{
+}
+
+int
+PyOS_InterruptOccurred(void)
+{
+    _wyield();
+}
+
+#define OK
+
+#endif /* QUICKWIN */
+
+#if defined(_M_IX86) && !defined(__QNX__)
+#include <io.h>
+#endif
+
+#if defined(MSDOS) && !defined(QUICKWIN)
+
+#ifdef __GNUC__
+
+/* This is for DJGPP's GO32 extender.  I don't know how to trap
+ * control-C  (There's no API for ctrl-C, and I don't want to mess with
+ * the interrupt vectors.)  However, this DOES catch control-break.
+ * --Amrit
+ */
+
+#include <go32.h>
+
+void
+PyOS_InitInterrupts(void)
+{
+    _go32_want_ctrl_break(1 /* TRUE */);
+}
+
+void
+PyOS_FiniInterrupts(void)
+{
+}
+
+int
+PyOS_InterruptOccurred(void)
+{
+    return _go32_was_ctrl_break_hit();
+}
+
+#else /* !__GNUC__ */
+
+/* This might work for MS-DOS (untested though): */
+
+void
+PyOS_InitInterrupts(void)
+{
+}
+
+void
+PyOS_FiniInterrupts(void)
+{
+}
+
+int
+PyOS_InterruptOccurred(void)
+{
+    int interrupted = 0;
+    while (kbhit()) {
+        if (getch() == '\003')
+            interrupted = 1;
+    }
+    return interrupted;
+}
+
+#endif /* __GNUC__ */
+
+#define OK
+
+#endif /* MSDOS && !QUICKWIN */
+
+
+#ifndef OK
+
+/* Default version -- for real operating systems and for Standard C */
+
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+
+static int interrupted;
+
+void
+PyErr_SetInterrupt(void)
+{
+    interrupted = 1;
+}
+
+extern int PyErr_CheckSignals(void);
+
+static int
+checksignals_witharg(void * arg)
+{
+    return PyErr_CheckSignals();
+}
+
+static void
+intcatcher(int sig)
+{
+    extern void Py_Exit(int);
+    static char message[] =
+"python: to interrupt a truly hanging Python program, interrupt once more.\n";
+    switch (interrupted++) {
+    case 0:
+        break;
+    case 1:
+#ifdef RISCOS
+        fprintf(stderr, message);
+#else
+        write(2, message, strlen(message));
+#endif
+        break;
+    case 2:
+        interrupted = 0;
+        Py_Exit(1);
+        break;
+    }
+    PyOS_setsig(SIGINT, intcatcher);
+    Py_AddPendingCall(checksignals_witharg, NULL);
+}
+
+static void (*old_siginthandler)(int) = SIG_DFL;
+
+void
+PyOS_InitInterrupts(void)
+{
+    if ((old_siginthandler = PyOS_setsig(SIGINT, SIG_IGN)) != SIG_IGN)
+        PyOS_setsig(SIGINT, intcatcher);
+}
+
+void
+PyOS_FiniInterrupts(void)
+{
+    PyOS_setsig(SIGINT, old_siginthandler);
+}
+
+int
+PyOS_InterruptOccurred(void)
+{
+    if (!interrupted)
+        return 0;
+    interrupted = 0;
+    return 1;
+}
+
+#endif /* !OK */
+
+void
+PyOS_AfterFork(void)
+{
+#ifdef WITH_THREAD
+    PyEval_ReInitThreads();
+    PyThread_ReInitTLS();
+#endif
+}

from_cpython/Parser/listnode.c

+
+/* List a node on a file */
+
+#include "pgenheaders.h"
+#include "token.h"
+#include "node.h"
+
+/* Forward */
+static void list1node(FILE *, node *);
+static void listnode(FILE *, node *);
+
+void
+PyNode_ListTree(node *n)
+{
+    listnode(stdout, n);
+}
+
+static int level, atbol;
+
+static void
+listnode(FILE *fp, node *n)
+{
+    level = 0;
+    atbol = 1;
+    list1node(fp, n);
+}
+
+static void
+list1node(FILE *fp, node *n)
+{
+    if (n == 0)
+        return;
+    if (ISNONTERMINAL(TYPE(n))) {
+        int i;
+        for (i = 0; i < NCH(n); i++)
+            list1node(fp, CHILD(n, i));
+    }
+    else if (ISTERMINAL(TYPE(n))) {
+        switch (TYPE(n)) {
+        case INDENT:
+            ++level;
+            break;
+        case DEDENT:
+            --level;
+            break;
+        default:
+            if (atbol) {
+                int i;
+                for (i = 0; i < level; ++i)
+                    fprintf(fp, "\t");
+                atbol = 0;
+            }
+            if (TYPE(n) == NEWLINE) {
+                if (STR(n) != NULL)
+                    fprintf(fp, "%s", STR(n));
+                fprintf(fp, "\n");
+                atbol = 1;
+            }
+            else
+                fprintf(fp, "%s ", STR(n));
+            break;
+        }
+    }
+    else
+        fprintf(fp, "? ");
+}

from_cpython/Parser/metagrammar.c

+
+#include "pgenheaders.h"
+#include "metagrammar.h"
+#include "grammar.h"
+#include "pgen.h"
+static arc arcs_0_0[3] = {
+    {2, 0},
+    {3, 0},
+    {4, 1},
+};
+static arc arcs_0_1[1] = {
+    {0, 1},
+};
+static state states_0[2] = {
+    {3, arcs_0_0},
+    {1, arcs_0_1},
+};
+static arc arcs_1_0[1] = {
+    {5, 1},
+};
+static arc arcs_1_1[1] = {
+    {6, 2},
+};
+static arc arcs_1_2[1] = {
+    {7, 3},
+};
+static arc arcs_1_3[1] = {
+    {3, 4},
+};
+static arc arcs_1_4[1] = {
+    {0, 4},
+};
+static state states_1[5] = {
+    {1, arcs_1_0},
+    {1, arcs_1_1},
+    {1, arcs_1_2},
+    {1, arcs_1_3},
+    {1, arcs_1_4},
+};
+static arc arcs_2_0[1] = {
+    {8, 1},
+};
+static arc arcs_2_1[2] = {
+    {9, 0},
+    {0, 1},
+};
+static state states_2[2] = {
+    {1, arcs_2_0},
+    {2, arcs_2_1},
+};
+static arc arcs_3_0[1] = {
+    {10, 1},
+};
+static arc arcs_3_1[2] = {
+    {10, 1},
+    {0, 1},
+};
+static state states_3[2] = {
+    {1, arcs_3_0},
+    {2, arcs_3_1},
+};
+static arc arcs_4_0[2] = {
+    {11, 1},
+    {13, 2},
+};
+static arc arcs_4_1[1] = {
+    {7, 3},
+};
+static arc arcs_4_2[3] = {
+    {14, 4},
+    {15, 4},
+    {0, 2},
+};
+static arc arcs_4_3[1] = {
+    {12, 4},
+};
+static arc arcs_4_4[1] = {
+    {0, 4},
+};
+static state states_4[5] = {
+    {2, arcs_4_0},
+    {1, arcs_4_1},
+    {3, arcs_4_2},
+    {1, arcs_4_3},
+    {1, arcs_4_4},
+};
+static arc arcs_5_0[3] = {
+    {5, 1},
+    {16, 1},
+    {17, 2},
+};
+static arc arcs_5_1[1] = {
+    {0, 1},
+};
+static arc arcs_5_2[1] = {
+    {7, 3},
+};
+static arc arcs_5_3[1] = {
+    {18, 1},
+};
+static state states_5[4] = {
+    {3, arcs_5_0},
+    {1, arcs_5_1},
+    {1, arcs_5_2},
+    {1, arcs_5_3},
+};
+static dfa dfas[6] = {
+    {256, "MSTART", 0, 2, states_0,
+     "\070\000\000"},
+    {257, "RULE", 0, 5, states_1,
+     "\040\000\000"},
+    {258, "RHS", 0, 2, states_2,
+     "\040\010\003"},
+    {259, "ALT", 0, 2, states_3,
+     "\040\010\003"},
+    {260, "ITEM", 0, 5, states_4,
+     "\040\010\003"},
+    {261, "ATOM", 0, 4, states_5,
+     "\040\000\003"},
+};
+static label labels[19] = {
+    {0, "EMPTY"},
+    {256, 0},
+    {257, 0},
+    {4, 0},
+    {0, 0},
+    {1, 0},
+    {11, 0},
+    {258, 0},
+    {259, 0},
+    {18, 0},
+    {260, 0},
+    {9, 0},
+    {10, 0},
+    {261, 0},
+    {16, 0},
+    {14, 0},
+    {3, 0},
+    {7, 0},
+    {8, 0},
+};
+static grammar _PyParser_Grammar = {
+    6,
+    dfas,
+    {19, labels},
+    256
+};
+
+grammar *
+meta_grammar(void)
+{
+    return &_PyParser_Grammar;
+}
+
+grammar *
+Py_meta_grammar(void)
+{
+  return meta_grammar();
+}

from_cpython/Parser/node.c

+/* Parse tree node implementation */
+
+#include "Python.h"
+#include "node.h"
+#include "errcode.h"
+
+node *
+PyNode_New(int type)
+{
+    node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
+    if (n == NULL)
+        return NULL;
+    n->n_type = type;
+    n->n_str = NULL;
+    n->n_lineno = 0;
+    n->n_nchildren = 0;
+    n->n_child = NULL;
+    return n;
+}
+
+/* See comments at XXXROUNDUP below.  Returns -1 on overflow. */
+static int
+fancy_roundup(int n)
+{
+    /* Round up to the closest power of 2 >= n. */
+    int result = 256;
+    assert(n > 128);
+    while (result < n) {
+        result <<= 1;
+        if (result <= 0)
+            return -1;
+    }
+    return result;
+}
+
+/* A gimmick to make massive numbers of reallocs quicker.  The result is
+ * a number >= the input.  In PyNode_AddChild, it's used like so, when
+ * we're about to add child number current_size + 1:
+ *
+ *     if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1):
+ *         allocate space for XXXROUNDUP(current_size + 1) total children
+ *     else:
+ *         we already have enough space
+ *
+ * Since a node starts out empty, we must have
+ *
+ *     XXXROUNDUP(0) < XXXROUNDUP(1)
+ *
+ * so that we allocate space for the first child.  One-child nodes are very
+ * common (presumably that would change if we used a more abstract form
+ * of syntax tree), so to avoid wasting memory it's desirable that
+ * XXXROUNDUP(1) == 1.  That in turn forces XXXROUNDUP(0) == 0.
+ *
+ * Else for 2 <= n <= 128, we round up to the closest multiple of 4.  Why 4?
+ * Rounding up to a multiple of an exact power of 2 is very efficient, and
+ * most nodes with more than one child have <= 4 kids.
+ *
+ * Else we call fancy_roundup() to grow proportionately to n.  We've got an
+ * extreme case then (like test_longexp.py), and on many platforms doing
+ * anything less than proportional growth leads to exorbitant runtime
+ * (e.g., MacPython), or extreme fragmentation of user address space (e.g.,
+ * Win98).
+ *
+ * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
+ * reported that, with this scheme, 89% of PyObject_REALLOC calls in
+ * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually
+ * wastes very little memory, but is very effective at sidestepping
+ * platform-realloc disasters on vulnerable platforms.
+ *
+ * Note that this would be straightforward if a node stored its current
+ * capacity.  The code is tricky to avoid that.
+ */
+#define XXXROUNDUP(n) ((n) <= 1 ? (n) :                 \
+               (n) <= 128 ? (((n) + 3) & ~3) :          \
+               fancy_roundup(n))
+
+
+int
+PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
+{
+    const int nch = n1->n_nchildren;
+    int current_capacity;
+    int required_capacity;
+    node *n;
+
+    if (nch == INT_MAX || nch < 0)
+        return E_OVERFLOW;
+
+    current_capacity = XXXROUNDUP(nch);
+    required_capacity = XXXROUNDUP(nch + 1);
+    if (current_capacity < 0 || required_capacity < 0)
+        return E_OVERFLOW;
+    if (current_capacity < required_capacity) {
+        if (required_capacity > PY_SIZE_MAX / sizeof(node)) {
+            return E_NOMEM;
+        }
+        n = n1->n_child;
+        n = (node *) PyObject_REALLOC(n,
+                                      required_capacity * sizeof(node));
+        if (n == NULL)
+            return E_NOMEM;
+        n1->n_child = n;
+    }
+
+    n = &n1->n_child[n1->n_nchildren++];
+    n->n_type = type;
+    n->n_str = str;
+    n->n_lineno = lineno;
+    n->n_col_offset = col_offset;
+    n->n_nchildren = 0;
+    n->n_child = NULL;
+    return 0;
+}
+
+/* Forward */
+static void freechildren(node *);
+static Py_ssize_t sizeofchildren(node *n);
+
+
+void
+PyNode_Free(node *n)
+{
+    if (n != NULL) {
+        freechildren(n);
+        PyObject_FREE(n);
+    }
+}
+
+Py_ssize_t
+_PyNode_SizeOf(node *n)
+{
+    Py_ssize_t res = 0;
+
+    if (n != NULL)
+        res = sizeof(node) + sizeofchildren(n);
+    return res;
+}
+
+static void
+freechildren(node *n)
+{
+    int i;
+    for (i = NCH(n); --i >= 0; )
+        freechildren(CHILD(n, i));
+    if (n->n_child != NULL)
+        PyObject_FREE(n->n_child);
+    if (STR(n) != NULL)
+        PyObject_FREE(STR(n));
+}
+
+static Py_ssize_t
+sizeofchildren(node *n)
+{
+    Py_ssize_t res = 0;
+    int i;
+    for (i = NCH(n); --i >= 0; )
+        res += sizeofchildren(CHILD(n, i));
+    if (n->n_child != NULL)
+        /* allocated size of n->n_child array */
+        res += XXXROUNDUP(NCH(n)) * sizeof(node);
+    if (STR(n) != NULL)
+        res += strlen(STR(n)) + 1;
+    return res;
+}

from_cpython/Parser/parser.h

+#ifndef Py_PARSER_H
+#define Py_PARSER_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Parser interface */
+
+#define MAXSTACK 1500
+
+typedef struct {
+	int		 s_state;	/* State in current DFA */
+	dfa		*s_dfa;		/* Current DFA */
+	struct _node	*s_parent;	/* Where to add next node */
+} stackentry;
+
+typedef struct {
+	stackentry	*s_top;		/* Top entry */
+	stackentry	 s_base[MAXSTACK];/* Array of stack entries */
+					/* NB The stack grows down */
+} stack;
+
+typedef struct {
+	stack	 	p_stack;	/* Stack of parser states */
+	grammar		*p_grammar;	/* Grammar to use */
+	node		*p_tree;	/* Top of parse tree */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	unsigned long	p_flags;	/* see co_flags in Include/code.h */
+#endif
+} parser_state;
+
+parser_state *PyParser_New(grammar *g, int start);
+void PyParser_Delete(parser_state *ps);
+int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
+                      int *expected_ret);
+void PyGrammar_AddAccelerators(grammar *g);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PARSER_H */

from_cpython/Parser/printgrammar.c

+
+/* Print a bunch of C initializers that represent a grammar */
+
+#include "pgenheaders.h"
+#include "grammar.h"
+
+/* Forward */
+static void printarcs(int, dfa *, FILE *);
+static void printstates(grammar *, FILE *);
+static void printdfas(grammar *, FILE *);
+static void printlabels(grammar *, FILE *);
+
+void
+printgrammar(grammar *g, FILE *fp)
+{
+    fprintf(fp, "/* Generated by Parser/pgen */\n\n");
+    fprintf(fp, "#include \"pgenheaders.h\"\n");
+    fprintf(fp, "#include \"grammar.h\"\n");
+    fprintf(fp, "PyAPI_DATA(grammar) _PyParser_Grammar;\n");
+    printdfas(g, fp);
+    printlabels(g, fp);
+    fprintf(fp, "grammar _PyParser_Grammar = {\n");
+    fprintf(fp, "    %d,\n", g->g_ndfas);
+    fprintf(fp, "    dfas,\n");
+    fprintf(fp, "    {%d, labels},\n", g->g_ll.ll_nlabels);
+    fprintf(fp, "    %d\n", g->g_start);
+    fprintf(fp, "};\n");
+}
+
+void
+printnonterminals(grammar *g, FILE *fp)
+{
+    dfa *d;
+    int i;
+
+    fprintf(fp, "/* Generated by Parser/pgen */\n\n");
+
+    d = g->g_dfa;
+    for (i = g->g_ndfas; --i >= 0; d++)
+        fprintf(fp, "#define %s %d\n", d->d_name, d->d_type);
+}
+
+static void
+printarcs(int i, dfa *d, FILE *fp)
+{
+    arc *a;
+    state *s;
+    int j, k;
+
+    s = d->d_state;
+    for (j = 0; j < d->d_nstates; j++, s++) {
+        fprintf(fp, "static arc arcs_%d_%d[%d] = {\n",
+            i, j, s->s_narcs);
+        a = s->s_arc;
+        for (k = 0; k < s->s_narcs; k++, a++)
+            fprintf(fp, "    {%d, %d},\n", a->a_lbl, a->a_arrow);
+        fprintf(fp, "};\n");
+    }
+}
+
+static void
+printstates(grammar *g, FILE *fp)
+{
+    state *s;
+    dfa *d;
+    int i, j;
+
+    d = g->g_dfa;
+    for (i = 0; i < g->g_ndfas; i++, d++) {
+        printarcs(i, d, fp);
+        fprintf(fp, "static state states_%d[%d] = {\n",
+            i, d->d_nstates);
+        s = d->d_state;
+        for (j = 0; j < d->d_nstates; j++, s++)
+            fprintf(fp, "    {%d, arcs_%d_%d},\n",
+                s->s_narcs, i, j);
+        fprintf(fp, "};\n");
+    }
+}
+
+static void
+printdfas(grammar *g, FILE *fp)
+{
+    dfa *d;
+    int i, j;
+
+    printstates(g, fp);
+    fprintf(fp, "static dfa dfas[%d] = {\n", g->g_ndfas);
+    d = g->g_dfa;
+    for (i = 0; i < g->g_ndfas; i++, d++) {
+        fprintf(fp, "    {%d, \"%s\", %d, %d, states_%d,\n",
+            d->d_type, d->d_name, d->d_initial, d->d_nstates, i);
+        fprintf(fp, "     \"");
+        for (j = 0; j < NBYTES(g->g_ll.ll_nlabels); j++)
+            fprintf(fp, "\\%03o", d->d_first[j] & 0xff);
+        fprintf(fp, "\"},\n");
+    }
+    fprintf(fp, "};\n");
+}
+
+static void
+printlabels(grammar *g, FILE *fp)
+{
+    label *l;
+    int i;
+
+    fprintf(fp, "static label labels[%d] = {\n", g->g_ll.ll_nlabels);
+    l = g->g_ll.ll_label;
+    for (i = g->g_ll.ll_nlabels; --i >= 0; l++) {
+        if (l->lb_str == NULL)
+            fprintf(fp, "    {%d, 0},\n", l->lb_type);
+        else
+            fprintf(fp, "    {%d, \"%s\"},\n",
+                l->lb_type, l->lb_str);
+    }
+    fprintf(fp, "};\n");
+}

from_cpython/Parser/tokenizer_pgen.c

+#define PGEN
+#include "tokenizer.c"

src/CMakeLists.txt

@@ -38,6 +38,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
 		codegen/baseline_jit.cpp
 		codegen/codegen.cpp
 		codegen/compvars.cpp
+        codegen/cpython_ast.cpp
 		codegen/entry.cpp
 		codegen/gcbuilder.cpp
 		codegen/irgen.cpp

src/core/ast.cpp

@@ -2232,4 +2232,17 @@ void flatten(AST_expr* root, std::vector<AST*>& output, bool expand_scopes) {
 
     root->accept(&visitor);
 }
+
+void makeModuleInteractive(AST_Module* m) {
+    for (int i = 0; i < m->body.size(); ++i) {
+        AST_stmt* s = m->body[i];
+        if (s->type != AST_TYPE::Expr)
+            continue;
+
+        AST_Expr* expr = (AST_Expr*)s;
+        AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
+        print_expr->args.push_back(expr->value);
+        expr->value = print_expr;
+    }
+}
 }