Merge commit '875432' into refcounting

wip -- tricky merge

from_cpython/CMakeLists.txt

@@ -92,6 +92,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
     exceptions.c
     floatobject.c
     fileobject.c
+    import.c
     iterobject.c
     memoryobject.c
     obmalloc.c

from_cpython/Include/import.h

@@ -11,7 +11,7 @@ extern "C" {
 PyAPI_FUNC(long) PyImport_GetMagicNumber(void) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(const char *name, PyObject *co) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx(
-	char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
+    const char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_AddModule(const char *name) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ImportModule(const char *name) PYSTON_NOEXCEPT;
@@ -26,7 +26,7 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m) PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyImport_Cleanup(void) PYSTON_NOEXCEPT;
-PyAPI_FUNC(int) PyImport_ImportFrozenModule(char *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyImport_ImportFrozenModule(const char *) PYSTON_NOEXCEPT;
 
 #ifdef WITH_THREAD
 PyAPI_FUNC(void) _PyImport_AcquireLock(void) PYSTON_NOEXCEPT;
@@ -45,12 +45,11 @@ PyAPI_FUNC(PyObject *) _PyImport_FindExtension(char *, char *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _PyImport_FixupExtension(char *, char *) PYSTON_NOEXCEPT;
 
 struct _inittab {
-    char *name;
+    const char *name;
     void (*initfunc)(void);
 };
 
-// Pyston change: this is no longer a static object
-//PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
+PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
 PyAPI_DATA(struct _inittab *) PyImport_Inittab;
 
 PyAPI_FUNC(int) PyImport_AppendInittab(const char *name, void (*initfunc)(void)) PYSTON_NOEXCEPT;

from_cpython/Lib/test/test_atexit.py

-# expected: fail
 import sys
 import unittest
 import StringIO

from_cpython/Python/importdl.h

+#ifndef Py_IMPORTDL_H
+#define Py_IMPORTDL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Definitions for dynamic loading of extension modules */
+enum filetype {
+    SEARCH_ERROR,
+    PY_SOURCE,
+    PY_COMPILED,
+    C_EXTENSION,
+    PY_RESOURCE, /* Mac only */
+    PKG_DIRECTORY,
+    C_BUILTIN,
+    PY_FROZEN,
+    PY_CODERESOURCE, /* Mac only */
+    IMP_HOOK
+};
+
+struct filedescr {
+    char *suffix;
+    char *mode;
+    enum filetype type;
+};
+extern struct filedescr * _PyImport_Filetab;
+extern const struct filedescr _PyImport_DynLoadFiletab[];
+
+extern PyObject *_PyImport_LoadDynamicModule(char *name, char *pathname,
+                                             FILE *);
+
+/* Max length of module suffix searched for -- accommodates "module.slb" */
+#define MAXSUFFIXSIZE 12
+
+#ifdef MS_WINDOWS
+#include <windows.h>
+typedef FARPROC dl_funcptr;
+#else
+#if defined(PYOS_OS2) && !defined(PYCC_GCC)
+#include <os2def.h>
+typedef int (* APIENTRY dl_funcptr)();
+#else
+typedef void (*dl_funcptr)(void);
+#endif
+#endif
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_IMPORTDL_H */

src/capi/modsupport.cpp

@@ -433,7 +433,7 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
 
 extern "C" PyObject* PyModule_GetDict(BORROWED(PyObject*) _m) noexcept {
     BoxedModule* m = static_cast<BoxedModule*>(_m);
-    assert(m->cls == module_cls);
+    assert(PyModule_Check(m));
 
     return autoDecref(m->getAttrWrapper());
 }

src/jit.cpp

@@ -188,9 +188,10 @@ int handleArg(char code) {
         TRAP = true;
     else if (code == 'q')
         GLOBAL_VERBOSITY = 0;
-    else if (code == 'v')
+    else if (code == 'v') {
+        Py_VerboseFlag++;
         GLOBAL_VERBOSITY++;
-    else if (code == 'd')
+    } else if (code == 'd')
         SHOW_DISASM = true;
     else if (code == 'I')
         FORCE_INTERPRETER = true;
@@ -276,7 +277,7 @@ static int RunMainFromImporter(const char* filename) {
     PyObject* argv0 = NULL, * importer = NULL;
 
     if ((argv0 = PyString_FromString(filename)) && (importer = PyImport_GetImporter(argv0))
-        && (importer->cls != null_importer_cls)) {
+        && (importer->cls != &PyNullImporter_Type)) {
         /* argv0 is usable as an import source, so
                put it in sys.path[0] and import __main__ */
         PyObject* sys_path = NULL;
@@ -438,8 +439,9 @@ static int main(int argc, char** argv) noexcept {
 
         if (!Py_NoSiteFlag) {
             try {
-                std::string module_name = "site";
-                importModuleLevel(module_name, None, None, 0);
+                Box* module = PyImport_ImportModule("site");
+                if (!module)
+                    throwCAPIException();
             } catch (ExcInfo e) {
                 e.printExcAndTraceback();
                 return 1;

src/runtime/builtin_modules/builtins.cpp

@@ -478,8 +478,11 @@ Box* bltinImport(Box* name, Box* globals, Box* locals, Box** args) {
         raiseExcHelper(TypeError, "an integer is required");
     }
 
-    std::string _name = static_cast<BoxedString*>(name)->s();
-    return importModuleLevel(_name, globals, fromlist, ((BoxedInt*)level)->n);
+    Box* rtn
+        = PyImport_ImportModuleLevel(((BoxedString*)name)->c_str(), globals, NULL, fromlist, ((BoxedInt*)level)->n);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
 }
 
 Box* delattrFunc(Box* obj, Box* _str) {
@@ -2008,7 +2011,7 @@ void setupBuiltins() {
                                    ParamNames({ "name", "globals", "locals", "fromlist", "level" }, "", ""));
     builtins_module->giveAttr(
         "__import__", new BoxedBuiltinFunctionOrMethod(import_func, "__import__",
-                                                       { None, None, None, autoDecref(boxInt(-1)) }, NULL, import_doc));
+                                                       { NULL, NULL, NULL, autoDecref(boxInt(-1)) }, NULL, import_doc));
 
     enumerate_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedEnumerate), false, "enumerate", false,
                                        BoxedEnumerate::dealloc, NULL, true, BoxedEnumerate::traverse, NOCLEAR);

src/runtime/builtin_modules/sys.cpp

@@ -82,7 +82,7 @@ BoxedDict* getSysModulesDict() {
     // return static_cast<BoxedDict*>(_sys_modules);
 
     // CPython's behavior: return an internalized reference:
-    return sys_modules_dict;
+    return incref(sys_modules_dict);
 }
 
 BoxedList* getSysPath() {
@@ -692,10 +692,6 @@ void setupSys() {
                                                   FunctionMetadata::create((void*)sysGetRecursionLimit, UNKNOWN, 0),
                                                   "getrecursionlimit", getrecursionlimit_doc));
 
-    sys_module->giveAttr("meta_path", new BoxedList());
-    sys_module->giveAttr("path_hooks", new BoxedList());
-    sys_module->giveAttr("path_importer_cache", new BoxedDict());
-
     // As we don't support compile() etc yet force 'dont_write_bytecode' to true.
     sys_module->giveAttrBorrowed("dont_write_bytecode", True);
 
@@ -755,9 +751,8 @@ void setupSys() {
 
 void setupSysEnd() {
     std::vector<Box*> builtin_module_names;
-    for (const auto& p : *sys_modules_dict) {
-        builtin_module_names.push_back(p.first);
-    }
+    for (int i = 0; PyImport_Inittab[i].name != NULL; i++)
+        builtin_module_names.push_back(boxString(PyImport_Inittab[i].name));
 
     std::sort<decltype(builtin_module_names)::iterator, PyLt>(builtin_module_names.begin(), builtin_module_names.end(),
                                                               PyLt());
@@ -765,6 +760,21 @@ void setupSysEnd() {
     sys_module->giveAttr("builtin_module_names",
                          BoxedTuple::create(builtin_module_names.size(), &builtin_module_names[0]));
 
+#ifndef NDEBUG
+    for (const auto& p : *sys_modules_dict) {
+        assert(PyString_Check(p.first));
+
+        bool found = false;
+        for (int i = 0; PyImport_Inittab[i].name != NULL; i++) {
+            if (((BoxedString*)p.first)->s() == PyImport_Inittab[i].name) {
+                found = true;
+            }
+        }
+        if (!found)
+            assert(0 && "found a module which is inside sys.modules but not listed inside PyImport_Inittab!");
+    }
+#endif
+
     /* version_info */
     if (VersionInfoType.tp_name == 0)
         PyStructSequence_InitType((PyTypeObject*)&VersionInfoType, &version_info_desc);

src/runtime/capi.cpp

@@ -1426,17 +1426,6 @@ extern "C" void makePendingCalls() {
         throwCAPIException();
 }
 
-extern "C" PyObject* _PyImport_FixupExtension(char* name, char* filename) noexcept {
-    // Don't have to do anything here, since we will error in _PyImport_FindExtension
-    // TODO is this ok?
-    return NULL;
-}
-
-extern "C" PyObject* _PyImport_FindExtension(char* name, char* filename) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
-}
-
 static PyObject* listmethodchain(PyMethodChain* chain) noexcept {
     PyMethodChain* c;
     PyMethodDef* ml;

src/runtime/classobj.cpp

@@ -300,6 +300,11 @@ static Box* classobjSetattr(Box* _cls, Box* _attr, Box* _value) {
     return None;
 }
 
+static Box* classobjDelattr(Box* _cls, Box* _attr) {
+    _classobjSetattr(_cls, _attr, NULL);
+    return None;
+}
+
 static int classobj_setattro(Box* cls, Box* attr, Box* value) noexcept {
     try {
         _classobjSetattr(cls, attr, value);
@@ -1794,6 +1799,8 @@ void setupClassobj() {
                            new BoxedFunction(FunctionMetadata::create((void*)classobjGetattribute, UNKNOWN, 2)));
     classobj_cls->giveAttr("__setattr__",
                            new BoxedFunction(FunctionMetadata::create((void*)classobjSetattr, UNKNOWN, 3)));
+    classobj_cls->giveAttr("__delattr__",
+                           new BoxedFunction(FunctionMetadata::create((void*)classobjDelattr, UNKNOWN, 2)));
     classobj_cls->giveAttr("__str__", new BoxedFunction(FunctionMetadata::create((void*)classobjStr, STR, 1)));
     classobj_cls->giveAttr("__repr__", new BoxedFunction(FunctionMetadata::create((void*)classobjRepr, STR, 1)));
     classobj_cls->giveAttrBorrowed("__dict__", dict_descr);

src/runtime/dict.cpp

@@ -188,8 +188,11 @@ Box* dictClear(BoxedDict* self) {
 }
 
 extern "C" PyObject* PyDict_Copy(PyObject* o) noexcept {
-    RELEASE_ASSERT(PyDict_Check(o), "");
+    RELEASE_ASSERT(PyDict_Check(o) || o->cls == attrwrapper_cls, "");
     try {
+        if (o->cls == attrwrapper_cls)
+            return attrwrapperToDict(o);
+
         return dictCopy(static_cast<BoxedDict*>(o));
     } catch (ExcInfo e) {
         setCAPIException(e);
@@ -694,15 +697,23 @@ void dictMergeFromSeq2(BoxedDict* self, Box* other) {
 }
 
 extern "C" int PyDict_Merge(PyObject* a, PyObject* b, int override_) noexcept {
-    if (a == NULL || !PyDict_Check(a) || b == NULL) {
-        PyErr_BadInternalCall();
-        return -1;
-    }
+    try {
+        if (a == NULL || !PyDict_Check(a) || b == NULL) {
+            if (a && b && a->cls == attrwrapper_cls) {
+                RELEASE_ASSERT(PyDict_Check(b) && override_ == 1, "");
+                for (auto&& item : *(BoxedDict*)b) {
+                    setitem(a, item.first, item.second);
+                }
+                return 0;
+            }
 
-    if (override_ != 1)
-        Py_FatalError("unimplemented");
+            PyErr_BadInternalCall();
+            return -1;
+        }
+
+        if (override_ != 1)
+            Py_FatalError("unimplemented");
 
-    try {
         dictMerge(static_cast<BoxedDict*>(a), b);
         return 0;
     } catch (ExcInfo e) {

src/runtime/import.h

@@ -21,7 +21,6 @@ namespace pyston {
 
 extern "C" PyObject* PyImport_GetImporter(PyObject* path) noexcept;
 extern "C" Box* import(int level, Box* from_imports, llvm::StringRef module_name);
-extern Box* importModuleLevel(llvm::StringRef module_name, Box* globals, Box* from_imports, int level);
 BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path);
 }
 

src/runtime/types.cpp

@@ -90,6 +90,7 @@ extern "C" void PyMarshal_Init();
 extern "C" void initstrop();
 extern "C" void initgc();
 extern "C" void init_ast();
+extern "C" void initimp();
 
 namespace pyston {
 
@@ -1372,8 +1373,8 @@ Box* dict_descr = NULL;
 extern "C" {
 BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls,
     * str_cls = NULL, *function_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls,
-      *member_descriptor_cls, *closure_cls, *generator_cls, *null_importer_cls, *complex_cls, *basestring_cls,
-      *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
+      *member_descriptor_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *property_cls,
+      *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
       *builtin_function_or_method_cls, *attrwrapperiter_cls, *set_cls, *frozenset_cls;
 
 BoxedTuple* EmptyTuple;
@@ -3747,6 +3748,60 @@ static int _check_and_flush(FILE* stream) {
     return fflush(stream) || prev_fail ? EOF : 0;
 }
 
+extern "C" {
+struct _inittab _PyImport_Inittab[] = { { "array", initarray },
+                                        { "_ast", init_ast },
+                                        { "binascii", initbinascii },
+                                        { "_codecs", init_codecs },
+                                        { "_collections", init_collections },
+                                        { "cStringIO", initcStringIO },
+                                        { "_csv", init_csv },
+                                        { "datetime", initdatetime },
+                                        { "errno", initerrno },
+                                        { "fcntl", initfcntl },
+                                        { "_functools", init_functools },
+                                        { "imp", initimp },
+                                        { "_io", init_io },
+                                        { "itertools", inititertools },
+                                        { "marshal", PyMarshal_Init },
+                                        { "math", initmath },
+                                        { "_md5", init_md5 },
+                                        { "operator", initoperator },
+                                        { "posix", initposix },
+                                        { "pwd", initpwd },
+                                        { "_random", init_random },
+                                        { "resource", initresource },
+                                        { "select", initselect },
+                                        { "_sha", init_sha },
+                                        { "_sha256", init_sha256 },
+                                        { "_sha512", init_sha512 },
+                                        { "signal", initsignal },
+                                        { "_socket", init_socket },
+                                        { "_sqlite3", init_sqlite3 },
+                                        { "_sre", init_sre },
+                                        { "_ssl", init_ssl },
+                                        { "strop", initstrop },
+                                        { "_struct", init_struct },
+                                        { "time", inittime },
+                                        { "unicodedata", initunicodedata },
+                                        { "_warnings", _PyWarnings_Init },
+                                        { "_weakref", init_weakref },
+                                        { "zipimport", initzipimport },
+                                        { "zlib", initzlib },
+
+                                        { "gc", initgc },
+                                        { "__pyston__", setupPyston },
+                                        { "thread", setupThread },
+
+                                        { "__main__", NULL },
+                                        { "__builtin__", NULL },
+                                        { "sys", NULL },
+                                        { "exceptions", NULL },
+
+
+                                        { 0, 0 } };
+}
+
 bool TRACK_ALLOCATIONS = false;
 void setupRuntime() {
     root_hcls = HiddenClass::makeRoot();
@@ -4252,10 +4307,8 @@ void setupRuntime() {
 
     setupBuiltins();
     _PyExc_Init();
-    setupThread();
-    setupImport();
-    initgc();
-    setupPyston();
+    _PyImport_Init();
+    _PyImportHooks_Init();
 
     PyType_Ready(&PyByteArrayIter_Type);
     PyType_Ready(&PyCapsule_Type);
@@ -4267,44 +4320,6 @@ void setupRuntime() {
     PyType_Ready(&PyDictProxy_Type);
     PyType_Ready(&PyTraceBack_Type);
 
-
-    initerrno();
-    init_sha();
-    init_sha256();
-    init_sha512();
-    init_md5();
-    init_random();
-    init_sre();
-    initmath();
-    initoperator();
-    initbinascii();
-    initpwd();
-    initposix();
-    init_struct();
-    initdatetime();
-    init_functools();
-    init_collections();
-    inititertools();
-    initresource();
-    initsignal();
-    initselect();
-    initfcntl();
-    inittime();
-    initarray();
-    initzlib();
-    init_codecs();
-    init_socket();
-    initunicodedata();
-    initcStringIO();
-    init_io();
-    initzipimport();
-    init_csv();
-    init_ssl();
-    init_sqlite3();
-    PyMarshal_Init();
-    initstrop();
-    init_ast();
-
     assert(object_cls->tp_setattro == PyObject_GenericSetAttr);
     assert(none_cls->tp_setattro == PyObject_GenericSetAttr);
 

src/runtime/types.h

@@ -77,10 +77,10 @@ extern "C" BoxedString* EmptyString;
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float_cls, *str_cls, *function_cls,
     *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *enumerate_cls,
-    *xrange_cls, *member_descriptor_cls, *null_importer_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls,
-    *basestring_cls, *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls,
-    *capi_getset_cls, *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls,
-    *wrapperdescr_cls, *wrapperobject_cls;
+    *xrange_cls, *member_descriptor_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
+    *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
+    *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls, *wrapperdescr_cls,
+    *wrapperobject_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
 #define memoryview_cls (&PyMemoryView_Type)

test/CPYTHON_TEST_NOTES.md

@@ -24,7 +24,6 @@ test_al                 No module named al
 test_applesingle        Not really a failure, but it tries to skip itself and we don't support that
 test_argparse           [unknown]
 test_ascii_formatd      segfault in ctypes (but only on CI)
-test_atexit             [unknown]
 test_audioop            [unknown]
 test_bigmem             [unknown]
 test_bisect             somehow sys.modules['_bisect'] is getting set to 0

test/tests/imp_test.py

@@ -13,7 +13,9 @@ for a in (1, "", "/proc", "nonexisting_dir"):
     except Exception as e:
         print e
 
+print imp.lock_held()
 imp.acquire_lock()
+print imp.lock_held()
 imp.release_lock()
 
 import os

test/tests/import_test.py

@@ -35,3 +35,9 @@ import _multiprocessing
 del _multiprocessing
 del sys.modules["_multiprocessing"]
 import _multiprocessing
+
+import time
+del time
+del sys.modules["time"]
+import time
+print time.sleep(0)