stdlib updates

- Add some more time functions
- Workaround for logging for now, to not try to show the caller frame
- Copy over everything in LIBPYSTON in the cmake build
  (useful for django to copy over things like templates)
- Allow str/unicode subclasses in more places
- Add a BoxedFile gc handler
- PyLong_FromString
- set.difference
- iter(iter(""))

from_cpython/Include/pyconfig.h

@@ -56,6 +56,8 @@
 #define HAVE_GETPEERNAME 1
 #define HAVE_STRFTIME 1
 #define HAVE_TIMES 1
+#define HAVE_STRUCT_TM_TM_ZONE 1
+#define HAVE_MKTIME 1
 
 #define PY_FORMAT_LONG_LONG "ll"
 #define PY_FORMAT_SIZE_T "z"

from_cpython/Lib/logging/__init__.py

@@ -1228,6 +1228,10 @@ class Logger(Filterer):
         Find the stack frame of the caller so that we can note the source
         file name, line number and function name.
         """
+
+        # Pyston change:
+        return "(unknown file)", 0, "(unknown function)"
+
         f = currentframe()
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.

lib_pyston/CMakeLists.txt

 # Copy any changed lib_pyston sources:
-file(GLOB_RECURSE LIBPYSTON_SRCS . "*.py")
+file(GLOB_RECURSE LIBPYSTON_SRCS . "*")
 set(LIBPYSTON_TARGETS "")
 foreach(STDLIB_FILE ${LIBPYSTON_SRCS})
     file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})

src/runtime/builtin_modules/builtins.cpp

@@ -516,7 +516,7 @@ Box* delattrFunc(Box* obj, Box* _str) {
 Box* getattrFunc(Box* obj, Box* _str, Box* default_value) {
     _str = coerceUnicodeToStr(_str);
 
-    if (_str->cls != str_cls) {
+    if (!isSubclass(_str->cls, str_cls)) {
         raiseExcHelper(TypeError, "getattr(): attribute name must be string");
     }
 

src/runtime/file.cpp

@@ -1358,6 +1358,18 @@ void fileDestructor(Box* b) {
     self->f_fp = NULL;
 }
 
+void BoxedFile::gcHandler(GCVisitor* v, Box* b) {
+    boxGCHandler(v, b);
+
+    assert(isSubclass(b->cls, file_cls));
+    BoxedFile* f = static_cast<BoxedFile*>(b);
+
+    v->visit(f->f_name);
+    v->visit(f->f_mode);
+    v->visit(f->f_encoding);
+    v->visit(f->f_errors);
+}
+
 void setupFile() {
     file_cls->simple_destructor = fileDestructor;
 

src/runtime/file.h

@@ -22,7 +22,8 @@ namespace pyston {
 
 class BoxedFile : public Box {
 public:
-    PyObject_HEAD FILE* f_fp;
+    PyObject_HEAD;
+    FILE* f_fp;
     PyObject* f_name;
     PyObject* f_mode;
     int (*f_close)(FILE*);
@@ -48,6 +49,8 @@ public:
         __attribute__((visibility("default")));
 
     DEFAULT_CLASS(file_cls);
+
+    static void gcHandler(GCVisitor* v, Box* b);
 };
 }
 

src/runtime/int.cpp

@@ -931,7 +931,7 @@ static Box* _intNew(Box* val, Box* base) {
         if (val->cls == int_cls)
             return n;
         return new BoxedInt(n->n);
-    } else if (val->cls == str_cls) {
+    } else if (isSubclass(val->cls, str_cls)) {
         int base_n;
         if (!base)
             base_n = 10;
@@ -947,7 +947,7 @@ static Box* _intNew(Box* val, Box* base) {
         if (!r)
             throwCAPIException();
         return r;
-    } else if (val->cls == unicode_cls) {
+    } else if (isSubclass(val->cls, unicode_cls)) {
         int base_n;
         if (!base)
             base_n = 10;

src/runtime/long.cpp

@@ -102,7 +102,15 @@ extern "C" PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject* obj, int* overflo
 }
 
 extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) noexcept {
-    Py_FatalError("unimplemented");
+    RELEASE_ASSERT(pend == NULL, "unsupported");
+
+    // See comment in _longNew
+    RELEASE_ASSERT(base >= 0, "unsupported");
+
+    BoxedLong* rtn = new BoxedLong();
+    int r = mpz_init_set_str(rtn->n, str, base);
+    RELEASE_ASSERT(r == 0, "");
+    return rtn;
 }
 
 static int64_t asSignedLong(BoxedLong* self) {

src/runtime/objmodel.cpp

@@ -1989,7 +1989,7 @@ extern "C" bool nonzero(Box* obj) {
     if (func == NULL) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == classobj_cls || obj->cls == type_cls
                    || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
-                   || obj->cls == instancemethod_cls || obj->cls == module_cls,
+                   || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls,
                "%s.__nonzero__", getTypeName(obj)); // TODO
 
         // TODO should rewrite these?
@@ -2109,7 +2109,7 @@ extern "C" BoxedInt* hash(Box* obj) {
 
     if (hash == NULL) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == function_cls || obj->cls == object_cls || obj->cls == classobj_cls
-                   || obj->cls == module_cls,
+                   || obj->cls == module_cls || obj->cls == capifunc_cls || obj->cls == instancemethod_cls,
                "%s.__hash__", getTypeName(obj));
         // TODO not the best way to handle this...
         return static_cast<BoxedInt*>(boxInt((i64)obj));

src/runtime/set.cpp

@@ -262,6 +262,22 @@ Box* setUnion(BoxedSet* self, BoxedTuple* args) {
     return rtn;
 }
 
+Box* setDifference(BoxedSet* self, BoxedTuple* args) {
+    if (!isSubclass(self->cls, set_cls) && !isSubclass(self->cls, frozenset_cls))
+        raiseExcHelper(TypeError, "descriptor 'difference' requires a 'set' object but received a '%s'",
+                       getTypeName(self));
+
+    BoxedSet* rtn = (BoxedSet*)setNew(self->cls, self);
+
+    for (auto container : args->pyElements()) {
+        for (auto elt : container->pyElements()) {
+            rtn->s.erase(elt);
+        }
+    }
+
+    return rtn;
+}
+
 static BoxedSet* setIntersection2(BoxedSet* self, Box* container) {
     assert(self->cls == set_cls);
 
@@ -458,6 +474,8 @@ void setupSet() {
     set_cls->giveAttr("union", new BoxedFunction(boxRTFunction((void*)setUnion, UNKNOWN, 1, 0, true, false)));
     set_cls->giveAttr("intersection",
                       new BoxedFunction(boxRTFunction((void*)setIntersection, UNKNOWN, 1, 0, true, false)));
+    set_cls->giveAttr("difference", new BoxedFunction(boxRTFunction((void*)setDifference, UNKNOWN, 1, 0, true, false)));
+    frozenset_cls->giveAttr("difference", set_cls->getattr("difference"));
     set_cls->giveAttr("issubset", new BoxedFunction(boxRTFunction((void*)setIssubset, UNKNOWN, 2)));
     set_cls->giveAttr("issuperset", new BoxedFunction(boxRTFunction((void*)setIssuperset, UNKNOWN, 2)));
 

src/runtime/types.cpp

@@ -1842,8 +1842,8 @@ void setupRuntime() {
     attrwrapper_cls = new BoxedHeapClass(object_cls, &AttrWrapper::gcHandler, 0, 0, sizeof(AttrWrapper), false,
                                          new BoxedString("attrwrapper"));
     dict_cls = new BoxedHeapClass(object_cls, &dictGCHandler, 0, 0, sizeof(BoxedDict), false, new BoxedString("dict"));
-    file_cls = new BoxedHeapClass(object_cls, NULL, 0, offsetof(BoxedFile, weakreflist), sizeof(BoxedFile), false,
-                                  new BoxedString("file"));
+    file_cls = new BoxedHeapClass(object_cls, &BoxedFile::gcHandler, 0, offsetof(BoxedFile, weakreflist),
+                                  sizeof(BoxedFile), false, new BoxedString("file"));
     int_cls = new BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedInt), false, new BoxedString("int"));
     bool_cls = new BoxedHeapClass(int_cls, NULL, 0, 0, sizeof(BoxedBool), false, new BoxedString("bool"));
     complex_cls = new BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedComplex), false, new BoxedString("complex"));

test/tests/long.py

@@ -98,3 +98,5 @@ x = I(D(C()))
 print type(x)
 
 print type(long(C()))
+
+print repr(int("123456789123456789123456789", 16))

test/tests/set.py

@@ -86,6 +86,11 @@ print sorted(s1.union(s2)), sorted(s1.intersection(s2))
 print sorted(s1.union(range(5, 7))), sorted(s1.intersection(range(5, 7)))
 print sorted(s2.union([], [], [], [])), sorted(s2.intersection())
 
+s = frozenset([1, 5])
+d = s.difference([1], [1], [2])
+print d, len(s)
+print
+
 l = []
 s = set(range(5))
 while s:
@@ -102,4 +107,4 @@ print s
 s = set(range(5))
 for i in xrange(10):
     s2 = set(range(i))
-    print s.issubset(s2), s.issuperset(s2), s == s2, s != s2
+    print s.issubset(s2), s.issuperset(s2), s == s2, s != s2, s.difference(s2)

test/tests/str_functions.py

@@ -159,3 +159,6 @@ print 'ab c\n\nde fg\rkl\r\n'.splitlines()
 print 'ab c\n\nde fg\rkl\r\n'.splitlines(True)
 
 print "1".zfill(3), "+1".zfill(3), "-1".zfill(3), "0".zfill(3)
+
+it = iter("hello world")
+print list(it)

test/tests/str_subclassing.py

@@ -6,3 +6,7 @@ print repr(s)
 
 import sys
 sys.stdout.write(s)
+print
+
+print repr("hello" + MyStr("world"))
+print int(MyStr("2"))

test/tests/time_test.py

@@ -4,3 +4,7 @@ print type(time)
 time.sleep(0)
 time.sleep(False)
 time.clock()
+
+print time.timezone
+
+print long(time.mktime(time.localtime(1020)))

test/tests/unicode_test.py

@@ -141,3 +141,5 @@ class C(object):
     def __repr__(self):
         return u"hello world"
 print [C()], set([C()]), {1:C()}
+
+print "hello" + u"world"