Inherit future flags during parsing

ie exec and eval statements are supposed to start with the future
flags of the scope in which they are called.  Our flags currently
get applied after parsing, which for the most part is fine, except
print_statement which actually changes the result of parsing.

ie `exec "print(1, 2)"` could either be parsed as a function call,
which will print "1 2", or as a print statement of a tuple, which
prints out "(1, 2)".  So thread the future flags through the parsing
stages as well.

libpypa @ 728c4bc8

-Subproject commit 1f62a285c57da50c00249a704deac00dae3c247b
+Subproject commit 728c4bc8c99d09da224ff3a2be82c98a1b026e82

src/codegen/irgen/hooks.cpp

@@ -370,12 +370,12 @@ static CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body
     return cl_f;
 }
 
-static AST_Module* parseExec(llvm::StringRef source, bool interactive = false) {
+static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, bool interactive = false) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
     const char* code = source.data();
-    AST_Module* parsedModule = parse_string(code);
+    AST_Module* parsedModule = parse_string(code, future_flags);
 
     if (interactive) {
         for (int i = 0; i < parsedModule->body.size(); ++i) {
@@ -397,7 +397,7 @@ static CLFunction* compileExec(AST_Module* parsedModule, BoxedString* fn, PyComp
     return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
 }
 
-static AST_Expression* parseEval(llvm::StringRef source) {
+static AST_Expression* parseEval(llvm::StringRef source, FutureFlags future_flags) {
     const char* code = source.data();
 
     // TODO error message if parse fails or if it isn't an expr
@@ -409,7 +409,7 @@ static AST_Expression* parseEval(llvm::StringRef source) {
     while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
         code++;
 
-    AST_Module* parsedModule = parse_string(code);
+    AST_Module* parsedModule = parse_string(code, future_flags);
     if (parsedModule->body.size() == 0)
         raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
 
@@ -496,11 +496,11 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
         llvm::StringRef source_str = static_cast<BoxedString*>(source)->s();
 
         if (type_str->s() == "exec") {
-            parsed = parseExec(source_str);
+            parsed = parseExec(source_str, future_flags);
         } else if (type_str->s() == "eval") {
-            parsed = parseEval(source_str);
+            parsed = parseEval(source_str, future_flags);
         } else if (type_str->s() == "single") {
-            parsed = parseExec(source_str, true);
+            parsed = parseExec(source_str, future_flags, true);
         } else {
             raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
         }
@@ -578,7 +578,11 @@ static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags*
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s());
+        CLFunction* caller_cl = getTopPythonFunction();
+        assert(caller_cl != NULL);
+        assert(caller_cl->source != NULL);
+
+        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileEval(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {
@@ -627,7 +631,7 @@ Box* execfile(Box* _fn, Box* globals, Box* locals) {
     if (!exists)
         raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s().data());
 
-    AST_Module* parsed = caching_parse_file(fn->s().data());
+    AST_Module* parsed = caching_parse_file(fn->s().data(), /* future_flags = */ 0);
     assert(parsed);
 
     CLFunction* caller_cl = getTopPythonFunction();
@@ -667,7 +671,11 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
+        CLFunction* caller_cl = getTopPythonFunction();
+        assert(caller_cl != NULL);
+        assert(caller_cl->source != NULL);
+
+        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileExec(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {

src/codegen/parser.cpp

@@ -1012,13 +1012,17 @@ static std::string getParserCommandLine(const char* fn) {
     return std::string("/usr/bin/python -S ") + parse_ast_fn.str().str() + " " + fn;
 }
 
-AST_Module* parse_string(const char* code) {
-    if (ENABLE_PYPA_PARSER) {
-        AST_Module* rtn = pypa_parse_string(code);
+AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
+    inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
+
+    if (ENABLE_PYPA_PARSER || inherited_flags) {
+        AST_Module* rtn = pypa_parse_string(code, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
         return rtn;
     }
 
+    ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
+
     int size = strlen(code);
     char buf[] = "pystontmp_XXXXXX";
     char* tmpdir = mkdtemp(buf);
@@ -1033,17 +1037,17 @@ AST_Module* parse_string(const char* code) {
     fputc('\n', f);
     fclose(f);
 
-    AST_Module* m = parse_file(tmp.c_str());
+    AST_Module* m = parse_file(tmp.c_str(), inherited_flags);
     removeDirectoryIfExists(tmpdir);
 
     return m;
 }
 
-AST_Module* parse_file(const char* fn) {
+AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
-        AST_Module* rtn = pypa_parse(fn);
+        AST_Module* rtn = pypa_parse(fn, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
         return rtn;
     }
@@ -1070,9 +1074,9 @@ AST_Module* parse_file(const char* fn) {
 
 const char* getMagic() {
     if (ENABLE_PYPA_PARSER)
-        return "a\ncM";
+        return "a\ncN";
     else
-        return "a\ncm";
+        return "a\ncn";
 }
 
 #define MAGIC_STRING_LENGTH 4
@@ -1080,7 +1084,9 @@ const char* getMagic() {
 #define CHECKSUM_LENGTH 1
 
 // Does at least one of: returns a valid file_data vector, or fills in 'module'
-static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module) {
+static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
+                                  FutureFlags inherited_flags) {
+    inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
     FILE* cache_fp = fopen(cache_fn.c_str(), "w");
 
     if (DEBUG_PARSING) {
@@ -1110,8 +1116,8 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
     file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
     checksum = 0;
 
-    if (ENABLE_PYPA_PARSER) {
-        module = pypa_parse(fn);
+    if (ENABLE_PYPA_PARSER || inherited_flags) {
+        module = pypa_parse(fn, inherited_flags);
         RELEASE_ASSERT(module, "unknown parse error");
 
         if (!cache_fp)
@@ -1121,6 +1127,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
         checksum = p.second;
         bytes_written += p.first;
     } else {
+        ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
         FILE* parser = popen(getParserCommandLine(fn).c_str(), "r");
         char buf[80];
         while (true) {
@@ -1157,7 +1164,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
 // Parsing the file is somewhat expensive since we have to shell out to cpython;
 // it's not a huge deal right now, but this caching version can significantly cut down
 // on the startup time (40ms -> 10ms).
-AST_Module* caching_parse_file(const char* fn) {
+AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
     std::ostringstream oss;
     if (DEBUG_PARSING) {
         oss << "caching_parse_file() on " << fn << '\n';
@@ -1192,7 +1199,7 @@ AST_Module* caching_parse_file(const char* fn) {
                     if (ferror(cache_fp)) {
                         oss << "encountered io error reading from the file\n";
                         AST_Module* mod = 0;
-                        file_data = _reparse(fn, cache_fn, mod);
+                        file_data = _reparse(fn, cache_fn, mod, inherited_flags);
                         if (mod)
                             return mod;
                         assert(file_data.size());
@@ -1295,7 +1302,7 @@ AST_Module* caching_parse_file(const char* fn) {
             file_data.clear();
 
             AST_Module* mod = 0;
-            file_data = _reparse(fn, cache_fn, mod);
+            file_data = _reparse(fn, cache_fn, mod, inherited_flags);
             if (mod)
                 return mod;
             assert(file_data.size());

src/codegen/parser.h

@@ -15,14 +15,15 @@
 #ifndef PYSTON_CODEGEN_PARSER_H
 #define PYSTON_CODEGEN_PARSER_H
 
+#include "core/types.h"
+
 namespace pyston {
 
 class AST_Module;
 
-AST_Module* parse_string(const char* code);
-
-AST_Module* parse_file(const char* fn);
-AST_Module* caching_parse_file(const char* fn);
+AST_Module* parse_string(const char* code, FutureFlags inherited_flags);
+AST_Module* parse_file(const char* fn, FutureFlags inherited_flags);
+AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags);
 }
 
 #endif

src/codegen/pypa-parser.cpp

@@ -1163,7 +1163,7 @@ private:
     int position;
 };
 
-static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader) {
+static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader, FutureFlags future_flags) {
     pypa::Lexer lexer(std::move(reader));
     pypa::SymbolTablePtr symbols;
     pypa::AstModulePtr module;
@@ -1176,6 +1176,35 @@ static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader) {
     options.error_handler = pypaErrorHandler;
     options.escape_handler = pypaEscapeDecoder;
 
+    if (future_flags & CO_FUTURE_PRINT_FUNCTION) {
+        future_flags &= ~CO_FUTURE_PRINT_FUNCTION;
+        options.initial_future_features.print_function = true;
+    }
+
+    if (future_flags & CO_FUTURE_DIVISION) {
+        future_flags &= ~CO_FUTURE_DIVISION;
+        options.initial_future_features.division = true;
+    }
+
+    if (future_flags & CO_FUTURE_ABSOLUTE_IMPORT) {
+        future_flags &= ~CO_FUTURE_ABSOLUTE_IMPORT;
+        options.initial_future_features.absolute_imports = true;
+    }
+
+    if (future_flags & CO_FUTURE_WITH_STATEMENT) {
+        future_flags &= ~CO_FUTURE_WITH_STATEMENT;
+        options.initial_future_features.with_statement = true;
+    }
+
+    if (future_flags & CO_FUTURE_UNICODE_LITERALS) {
+        future_flags &= ~CO_FUTURE_UNICODE_LITERALS;
+        options.initial_future_features.unicode_literals = true;
+    }
+
+    // Strip out some flags:
+    future_flags &= ~(CO_NESTED);
+    RELEASE_ASSERT(!future_flags, "0x%x", future_flags);
+
     if (pypa::parse(lexer, module, symbols, options) && module) {
         return readModule(*module);
     }
@@ -1199,17 +1228,17 @@ PyObject* PystonStringReader::open_python_file() noexcept {
     return PycStringIO->NewInput(s);
 }
 
-AST_Module* pypa_parse(char const* file_path) {
+AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags) {
     auto reader = PystonFileReader::create(file_path);
     if (!reader)
         return nullptr;
 
-    return parse_with_reader(std::move(reader));
+    return parse_with_reader(std::move(reader), future_flags);
 }
 
-AST_Module* pypa_parse_string(char const* str) {
+AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags) {
     auto reader = llvm::make_unique<PystonStringReader>(str);
 
-    return parse_with_reader(std::move(reader));
+    return parse_with_reader(std::move(reader), future_flags);
 }
 }

src/codegen/pypa-parser.h

@@ -17,10 +17,12 @@
 
 #include <cstdio>
 
+#include "core/types.h"
+
 namespace pyston {
 class AST_Module;
-AST_Module* pypa_parse(char const* file_path);
-AST_Module* pypa_parse_string(char const* str);
+AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags);
+AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags);
 }
 
 #endif // PYSTON_CODEGEN_PYPAPARSER_H

src/jit.cpp

@@ -418,7 +418,7 @@ static int main(int argc, char** argv) {
         if (command != NULL) {
             try {
                 main_module = createModule(boxString("__main__"), "<string>");
-                AST_Module* m = parse_string(command);
+                AST_Module* m = parse_string(command, /* future_flags = */ 0);
                 compileAndRunModule(m, main_module);
                 rtncode = 0;
             } catch (ExcInfo e) {
@@ -457,7 +457,7 @@ static int main(int argc, char** argv) {
 
                 main_module = createModule(boxString("__main__"), fn);
                 try {
-                    AST_Module* ast = caching_parse_file(fn);
+                    AST_Module* ast = caching_parse_file(fn, /* future_flags = */ 0);
                     compileAndRunModule(ast, main_module);
                 } catch (ExcInfo e) {
                     setCAPIException(e);
@@ -487,7 +487,7 @@ static int main(int argc, char** argv) {
                 add_history(line);
 
                 try {
-                    AST_Module* m = parse_string(line);
+                    AST_Module* m = parse_string(line, /* future_flags = */ 0);
 
                     Timer _t("repl");
 

src/runtime/import.cpp

@@ -43,7 +43,7 @@ static void removeModule(BoxedString* name) {
 Box* createAndRunModule(BoxedString* name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn.c_str());
 
-    AST_Module* ast = caching_parse_file(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
     assert(ast);
     try {
         compileAndRunModule(ast, module);
@@ -69,7 +69,7 @@ static Box* createAndRunModule(BoxedString* name, const std::string& fn, const s
     static BoxedString* path_str = internStringImmortal("__path__");
     module->setattr(path_str, path_list, NULL);
 
-    AST_Module* ast = caching_parse_file(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
     assert(ast);
     try {
         compileAndRunModule(ast, module);
@@ -646,7 +646,7 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
 
         static BoxedString* file_str = internStringImmortal("__file__");
         module->setattr(file_str, boxString(pathname), NULL);
-        AST_Module* ast = parse_string(code->data());
+        AST_Module* ast = parse_string(code->data(), /* future_flags = */ 0);
         compileAndRunModule(ast, module);
         return module;
     } catch (ExcInfo e) {

test/tests/print_function.py

@@ -52,3 +52,9 @@ except TypeError:
 
 for i in xrange(10):
     print()
+
+# These two should output "1 2"
+exec """print(1, 2)"""
+eval("print(1, 2)")
+# This one should output "(1, 2)"
+exec compile("print(1, 2)", "<string>", "exec", dont_inherit=True)

test/unittests/analysis.cpp

@@ -26,7 +26,7 @@ protected:
 
 TEST_F(AnalysisTest, augassign) {
     const std::string fn("test/unittests/analysis_listcomp.py");
-    AST_Module* module = caching_parse_file(fn.c_str());
+    AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
@@ -59,7 +59,7 @@ TEST_F(AnalysisTest, augassign) {
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     const std::string fn("test/unittests/analysis_osr.py");
-    AST_Module* module = caching_parse_file(fn.c_str());
+    AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     ScopingAnalysis *scoping = new ScopingAnalysis(module, true);