Merge pull request #509 from undingen/mysql

Add more imp.* functions, set correct module.__file__, import * with non module globals

from_cpython/Include/Python.h

@@ -110,6 +110,10 @@
 
 #include "pyfpe.h"
 
+#define Py_single_input 256
+#define Py_file_input 257
+#define Py_eval_input 258
+
 #ifdef __cplusplus
 extern "C" {
 #endif

from_cpython/Include/objimpl.h

@@ -311,8 +311,11 @@ extern PyGC_Head *_PyGC_generation0;
 #define _PyObject_GC_MAY_BE_TRACKED(obj) \
     (PyObject_IS_GC(obj) && \
         (!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj)))
-
+#else
+/* for source compatibility with 2.2 */
+#define _PyObject_GC_Del PyObject_GC_Del
 #endif
+
 PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t) PYSTON_NOEXCEPT;

src/capi/abstract.cpp

@@ -52,6 +52,12 @@ extern "C" int PyObject_Cmp(PyObject* o1, PyObject* o2, int* result) noexcept {
     return 0;
 }
 
+extern "C" PyObject* PyObject_Type(PyObject* o) noexcept {
+    if (o == NULL)
+        return null_error();
+    return o->cls;
+}
+
 extern "C" Py_ssize_t _PyObject_LengthHint(PyObject* o, Py_ssize_t defaultvalue) noexcept {
     static PyObject* hintstrobj = NULL;
     PyObject* ro, *hintmeth;

src/capi/modsupport.cpp

@@ -404,7 +404,7 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
         }
     }
 
-    BoxedModule* module = createModule(name, "__builtin__", doc);
+    BoxedModule* module = createModule(name, NULL, doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None
     Box* passthrough = static_cast<Box*>(self);

src/codegen/ast_interpreter.cpp

@@ -618,9 +618,7 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
 
         Value module = visit_expr(node->args[0]);
 
-        RELEASE_ASSERT(globals == source_info->parent_module,
-                       "'import *' currently not supported with overridden globals");
-        v = importStar(module.o, source_info->parent_module);
+        v = importStar(module.o, globals);
     } else if (node->opcode == AST_LangPrimitive::NONE) {
         v = None;
     } else if (node->opcode == AST_LangPrimitive::LANDINGPAD) {

src/codegen/irgen/hooks.cpp

@@ -607,6 +607,24 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
     return evalOrExec(cl, globals, locals);
 }
 
+extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
+                                       PyCompilerFlags* flags) noexcept {
+
+    try {
+        if (start == Py_file_input)
+            return exec(boxString(str), globals, locals);
+        else if (start == Py_eval_input)
+            return eval(boxString(str), globals, locals);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+
+    // Py_single_input is not yet implemented
+    RELEASE_ASSERT(0, "Unimplemented %d", start);
+    return 0;
+}
+
 // If a function version keeps failing its speculations, kill it (remove it
 // from the list of valid function versions).  The next time we go to call
 // the function, we will have to pick a different version, potentially recompiling.

src/core/types.h

@@ -623,7 +623,8 @@ class BoxedClass;
 void setupRuntime();
 void teardownRuntime();
 Box* createAndRunModule(const std::string& name, const std::string& fn);
-BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc = NULL);
+BoxedModule* createModule(const std::string& name, const char* fn = NULL, const char* doc = NULL);
+Box* moduleInit(BoxedModule* self, Box* name, Box* doc = NULL);
 
 // TODO where to put this
 void appendToSysPath(const std::string& path);

src/runtime/builtin_modules/builtins.cpp

@@ -1073,9 +1073,9 @@ Box* builtinApply(Box* func, Box* args, Box* keywords) {
 }
 
 void setupBuiltins() {
-    builtins_module = createModule("__builtin__", "__builtin__",
-                                   "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
-                                   "the `nil' object; Ellipsis represents `...' in slices.");
+    builtins_module
+        = createModule("__builtin__", NULL, "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
+                                            "the `nil' object; Ellipsis represents `...' in slices.");
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");

src/runtime/builtin_modules/gc.cpp

@@ -38,7 +38,7 @@ static Box* enable() {
 }
 
 void setupGC() {
-    BoxedModule* gc_module = createModule("gc", "__builtin__");
+    BoxedModule* gc_module = createModule("gc");
 
     gc_module->giveAttr("collect",
                         new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)gcCollect, NONE, 0), "collect"));

src/runtime/builtin_modules/pyston.cpp

@@ -62,7 +62,7 @@ static Box* dumpStats(Box* includeZeros) {
 }
 
 void setupPyston() {
-    pyston_module = createModule("__pyston__", "__builtin__");
+    pyston_module = createModule("__pyston__");
 
     pyston_module->giveAttr("setOption",
                             new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)setOption, UNKNOWN, 2), "setOption"));

src/runtime/builtin_modules/sys.cpp

@@ -386,7 +386,7 @@ void setupSys() {
     gc::registerPermanentRoot(sys_modules_dict);
 
     // This is ok to call here because we've already created the sys_modules_dict
-    sys_module = createModule("sys", "__builtin__");
+    sys_module = createModule("sys");
 
     sys_module->giveAttr("modules", sys_modules_dict);
 

src/runtime/builtin_modules/thread.cpp

@@ -225,7 +225,7 @@ Box* stackSize() {
 }
 
 void setupThread() {
-    thread_module = createModule("thread", "__builtin__");
+    thread_module = createModule("thread");
 
     thread_module->giveAttr("start_new_thread", new BoxedBuiltinFunctionOrMethod(
                                                     boxRTFunction((void*)startNewThread, BOXED_INT, 3, 1, false, false),

src/runtime/capi.cpp

@@ -1375,6 +1375,10 @@ extern "C" PyVarObject* _PyObject_GC_NewVar(PyTypeObject* tp, Py_ssize_t nitems)
     return op;
 }
 
+extern "C" void PyObject_GC_Del(void* op) noexcept {
+    PyObject_FREE(op);
+}
+
 extern "C" void _Py_FatalError(const char* fmt, const char* function, const char* message) {
     fprintf(stderr, fmt, function, message);
     fflush(stderr); /* it helps in Windows debug build */

src/runtime/import.cpp

@@ -40,7 +40,7 @@ static void removeModule(const std::string& name) {
 }
 
 Box* createAndRunModule(const std::string& name, const std::string& fn) {
-    BoxedModule* module = createModule(name, fn);
+    BoxedModule* module = createModule(name, fn.c_str());
 
     AST_Module* ast = caching_parse_file(fn.c_str());
     try {
@@ -57,7 +57,7 @@ Box* createAndRunModule(const std::string& name, const std::string& fn) {
 }
 
 static Box* createAndRunModule(const std::string& name, const std::string& fn, const std::string& module_path) {
-    BoxedModule* module = createModule(name, fn);
+    BoxedModule* module = createModule(name, fn.c_str());
 
     Box* b_path = boxStringPtr(&module_path);
 
@@ -613,7 +613,7 @@ extern "C" PyObject* PyImport_AddModule(const char* name) noexcept {
 
         if (m != NULL && m->cls == module_cls)
             return m;
-        return createModule(name, name);
+        return createModule(name);
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;
@@ -629,9 +629,9 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
         if (module == NULL)
             return NULL;
 
+        module->setattr("__file__", boxString(pathname), NULL);
         AST_Module* ast = parse_string(code->data());
         compileAndRunModule(ast, module);
-        module->setattr("__file__", boxString(pathname), NULL);
         return module;
     } catch (ExcInfo e) {
         removeModule(name);
@@ -708,7 +708,6 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
     Box* _description = args[0];
 
     RELEASE_ASSERT(_name->cls == str_cls, "");
-    RELEASE_ASSERT(_file == None, "");
     RELEASE_ASSERT(_pathname->cls == str_cls, "");
     RELEASE_ASSERT(_description->cls == tuple_cls, "");
 
@@ -721,15 +720,20 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
     BoxedString* mode = (BoxedString*)description->elts[1];
     BoxedInt* type = (BoxedInt*)description->elts[2];
 
-    RELEASE_ASSERT(suffix->cls == str_cls, "");
     RELEASE_ASSERT(mode->cls == str_cls, "");
     RELEASE_ASSERT(type->cls == int_cls, "");
-
-    RELEASE_ASSERT(suffix->s.empty(), "");
-    RELEASE_ASSERT(mode->s.empty(), "");
+    RELEASE_ASSERT(pathname->cls == str_cls, "");
+    RELEASE_ASSERT(pathname->size(), "");
 
     if (type->n == SearchResult::PKG_DIRECTORY) {
+        RELEASE_ASSERT(suffix->cls == str_cls, "");
+        RELEASE_ASSERT(suffix->s.empty(), "");
+        RELEASE_ASSERT(mode->s.empty(), "");
+        RELEASE_ASSERT(_file == None, "");
         return createAndRunModule(name->s, (llvm::Twine(pathname->s) + "/__init__.py").str(), pathname->s);
+    } else if (type->n == SearchResult::PY_SOURCE) {
+        RELEASE_ASSERT(_file->cls == file_cls, "");
+        return createAndRunModule(name->s, pathname->s);
     }
 
     Py_FatalError("unimplemented");
@@ -784,10 +788,38 @@ Box* impReleaseLock() {
     return None;
 }
 
+Box* impNewModule(Box* _name) {
+    if (!PyString_Check(_name))
+        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_name));
+
+    BoxedModule* module = new BoxedModule();
+    moduleInit(module, _name);
+    return module;
+}
+
+Box* impIsBuiltin(Box* _name) {
+    if (!PyString_Check(_name))
+        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_name));
+
+    BoxedTuple* builtin_modules = (BoxedTuple*)sys_module->getattr("builtin_module_names");
+    RELEASE_ASSERT(PyTuple_Check(builtin_modules), "");
+    for (Box* m : builtin_modules->pyElements()) {
+        if (compare(m, _name, AST_TYPE::Eq) == True)
+            return boxInt(-1); // CPython returns 1 for modules which can get reinitialized.
+    }
+    return boxInt(0);
+}
+
+Box* impIsFrozen(Box* name) {
+    if (!PyString_Check(name))
+        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(name));
+    return False;
+}
+
 void setupImport() {
     BoxedModule* imp_module
-        = createModule("imp", "__builtin__", "'This module provides the components needed to build your own\n"
-                                             "__import__ function.  Undocumented functions are obsolete.'");
+        = createModule("imp", NULL, "'This module provides the components needed to build your own\n"
+                                    "__import__ function.  Undocumented functions are obsolete.'");
 
     imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
     imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));
@@ -827,5 +859,12 @@ void setupImport() {
                                                                           "acquire_lock"));
     imp_module->giveAttr("release_lock", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)impReleaseLock, NONE, 0),
                                                                           "release_lock"));
+
+    imp_module->giveAttr("new_module",
+                         new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)impNewModule, MODULE, 1), "new_module"));
+    imp_module->giveAttr(
+        "is_builtin", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)impIsBuiltin, BOXED_INT, 1), "is_builtin"));
+    imp_module->giveAttr(
+        "is_frozen", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)impIsFrozen, BOXED_BOOL, 1), "is_frozen"));
 }
 }

src/runtime/objmodel.cpp

@@ -4863,13 +4863,9 @@ extern "C" Box* importFrom(Box* _m, const std::string* name) {
     raiseExcHelper(ImportError, "cannot import name %s", name->c_str());
 }
 
-extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
+extern "C" Box* importStar(Box* _from_module, Box* to_globals) {
     STAT_TIMER(t0, "us_timer_importStar");
 
-    // TODO(kmod): it doesn't seem too bad to update this to take custom globals;
-    // it looks like mostly a matter of changing the getattr calls to getitem.
-    RELEASE_ASSERT(getGlobals() == to_module, "importStar doesn't support custom globals yet");
-
     ASSERT(isSubclass(_from_module->cls, module_cls), "%s", _from_module->cls->tp_name);
     BoxedModule* from_module = static_cast<BoxedModule*>(_from_module);
 
@@ -4902,8 +4898,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
 
             if (!attr_value)
                 raiseExcHelper(AttributeError, "'module' object has no attribute '%s'", casted_attr_name->data());
-
-            to_module->setattr(casted_attr_name->s, attr_value, NULL);
+            setGlobal(to_globals, casted_attr_name->s, attr_value);
         }
         return None;
     }
@@ -4913,7 +4908,7 @@ extern "C" Box* importStar(Box* _from_module, BoxedModule* to_module) {
         if (p.first()[0] == '_')
             continue;
 
-        to_module->setattr(p.first(), module_attrs->attr_list->attrs[p.second], NULL);
+        setGlobal(to_globals, p.first(), module_attrs->attr_list->attrs[p.second]);
     }
 
     return None;

src/runtime/objmodel.h

@@ -79,7 +79,7 @@ extern "C" void delitem(Box* target, Box* slice);
 extern "C" Box* getclsattr(Box* obj, const char* attr);
 extern "C" Box* unaryop(Box* operand, int op_type);
 extern "C" Box* importFrom(Box* obj, const std::string* attr);
-extern "C" Box* importStar(Box* from_module, BoxedModule* to_module);
+extern "C" Box* importStar(Box* from_module, Box* to_globals);
 extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size);
 extern "C" void assertNameDefined(bool b, const char* name, BoxedClass* exc_cls, bool local_var_msg);
 extern "C" void assertFailDerefNameDefined(const char* name);

src/runtime/types.cpp

@@ -805,7 +805,8 @@ static Box* functionCall(BoxedFunction* self, Box* args, Box* kwargs) {
 static Box* funcName(Box* b, void*) {
     assert(b->cls == function_cls);
     BoxedFunction* func = static_cast<BoxedFunction*>(b);
-    RELEASE_ASSERT(func->name != NULL, "func->name is not set");
+    if (func->name == NULL)
+        return boxString("<unknown function name>");
     return func->name;
 }
 
@@ -1181,9 +1182,7 @@ static void typeSetModule(Box* _type, PyObject* value, void* context) {
 
 Box* typeHash(BoxedClass* self) {
     assert(isSubclass(self->cls, type_cls));
-
-    // This is how CPython defines it; seems reasonable enough:
-    return boxInt(reinterpret_cast<intptr_t>(self) >> 4);
+    return boxInt(_Py_HashPointer(self));
 }
 
 static PyObject* type_subclasses(PyTypeObject* type, PyObject* args_ignored) noexcept {
@@ -1680,6 +1679,10 @@ Box* objectStr(Box* obj) {
     return obj->reprIC();
 }
 
+Box* objectHash(Box* obj) {
+    return boxInt(_Py_HashPointer(obj));
+}
+
 Box* objectSetattr(Box* obj, Box* attr, Box* value) {
     attr = coerceUnicodeToStr(attr);
     if (attr->cls != str_cls) {
@@ -2339,6 +2342,8 @@ void setupRuntime() {
     object_cls->giveAttr("__init__", new BoxedFunction(boxRTFunction((void*)objectInit, UNKNOWN, 1, 0, true, false)));
     object_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)objectRepr, UNKNOWN, 1, 0, false, false)));
     object_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)objectStr, UNKNOWN, 1, 0, false, false)));
+    object_cls->giveAttr("__hash__",
+                         new BoxedFunction(boxRTFunction((void*)objectHash, BOXED_INT, 1, 0, false, false)));
     object_cls->giveAttr("__subclasshook__",
                          boxInstanceMethod(object_cls,
                                            new BoxedFunction(boxRTFunction((void*)objectSubclasshook, UNKNOWN, 2)),
@@ -2578,8 +2583,8 @@ void setupRuntime() {
     TRACK_ALLOCATIONS = true;
 }
 
-BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
-    assert(fn.size() && "probably wanted to set the fn to <stdin>?");
+BoxedModule* createModule(const std::string& name, const char* fn, const char* doc) {
+    assert(!fn || strlen(fn) && "probably wanted to set the fn to <stdin>?");
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
@@ -2593,7 +2598,8 @@ BoxedModule* createModule(const std::string& name, const std::string& fn, const
 
     BoxedModule* module = new BoxedModule();
     moduleInit(module, boxString(name), boxString(doc ? doc : ""));
-    module->giveAttr("__file__", boxString(fn));
+    if (fn)
+        module->giveAttr("__file__", boxString(fn));
 
     d->d[b_name] = module;
     return module;

test/tests/exec_in_test.py

@@ -156,3 +156,8 @@ l = types.ModuleType("TestMod2")
 exec ("global a; a=1; print a; b=2", g.__dict__, l.__dict__)
 print g.a
 print l.b
+
+s = "from sys import *"
+g = dict()
+exec s in g
+print "version" in g

test/tests/imp_test.py

@@ -22,3 +22,16 @@ m1 = imp.load_source("import_target", os.path.join(os.path.dirname(__file__), "i
 print "second load_source():"
 m2 = imp.load_source("import_target", os.path.join(os.path.dirname(__file__), "import_target.py"))
 print m1 is m2
+
+m = imp.new_module("My new module")
+print type(m), m, hasattr(m, "__file__")
+print imp.is_builtin("sys"), imp.is_frozen("sys")
+print imp.is_builtin("io"), imp.is_frozen("io")
+
+e = imp.find_module("1")
+m = imp.load_module("test_1", e[0], e[1], e[2])
+
+def n(s):
+    return str(s).replace(".pyc", ".py")
+
+print n(m), n(m.__name__), n(m.__file__), hasattr(m, "__path__")

test/tests/modules.py

@@ -8,6 +8,4 @@ x = repr(empty_module)
 print x[0:29]
 print x[-2:]
 
-# cpython 2.7.5 writes "from '/usr/lib64/python2.7/lib-dynload/math.so'"
-# pyston writes "(built-in)"
-print repr(math)[0:15] + "(built-in)>"
+print repr(math)

test/tests/object_default_behavior.py

@@ -30,3 +30,6 @@ del c.a
 del c.b
 del c.c
 print sorted(c.__dict__.items())
+
+v = 1
+print object.__hash__(v) == object.__hash__(v)

test/unittests/analysis.cpp

@@ -36,7 +36,7 @@ TEST_F(AnalysisTest, augassign) {
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body, fn);
+    SourceInfo* si = new SourceInfo(createModule("augassign", fn.c_str()), scoping, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
@@ -63,7 +63,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-    SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn),
+    SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn.c_str()),
             scoping, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);