Merge commit 'pr/466'

Conflicts:
	src/runtime/objmodel.cpp

Closes #466

src/analysis/scoping_analysis.cpp

@@ -20,6 +20,7 @@
 #include "core/common.h"
 #include "core/types.h"
 #include "core/util.h"
+#include "runtime/types.h"
 
 namespace pyston {
 
@@ -52,6 +53,30 @@ bool containsYield(AST* ast) {
     return visitor.containsYield;
 }
 
+// TODO
+// Combine this with the below? Basically the same logic with different string types...
+// Also should this go in this file?
+BoxedString* mangleNameBoxedString(BoxedString* id, BoxedString* private_name) {
+    assert(id);
+    assert(private_name);
+    int len = id->s.size();
+    if (len < 2 || id->s[0] != '_' || id->s[1] != '_')
+        return id;
+
+    if ((id->s[len - 2] == '_' && id->s[len - 1] == '_') || id->s.find('.') != llvm::StringRef::npos)
+        return id;
+
+    const char* p = private_name->s.data();
+    while (*p == '_') {
+        p++;
+        len--;
+    }
+    if (*p == '\0')
+        return id;
+
+    return static_cast<BoxedString*>(boxStringTwine("_" + (p + id->s)));
+}
+
 static void mangleNameInPlace(InternedString& id, const std::string* private_name,
                               InternedStringPool& interned_strings) {
     if (!private_name)

src/analysis/scoping_analysis.h

@@ -178,6 +178,9 @@ public:
 };
 
 bool containsYield(AST* ast);
+
+class BoxedString;
+BoxedString* mangleNameBoxedString(BoxedString* id, BoxedString* private_name);
 }
 
 #endif

src/core/types.h

@@ -443,10 +443,12 @@ public:
 
     llvm::iterator_range<BoxIterator> pyElements();
 
+    size_t getHCAttrsOffset();
     HCAttrs* getHCAttrsPtr();
     void setDict(BoxedDict* d);
     BoxedDict* getDict();
 
+
     void setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);
     void giveAttr(const std::string& attr, Box* val) {
         assert(!this->hasattr(attr));
@@ -476,9 +478,11 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
 
 #define DEFAULT_CLASS(default_cls)                                                                                     \
     void* operator new(size_t size, BoxedClass * cls) __attribute__((visibility("default"))) {                         \
+        assert(cls->tp_itemsize == 0);                                                                                 \
         return Box::operator new(size, cls);                                                                           \
     }                                                                                                                  \
     void* operator new(size_t size) __attribute__((visibility("default"))) {                                           \
+        assert(default_cls->tp_itemsize == 0);                                                                         \
         return Box::operator new(size, default_cls);                                                                   \
     }
 
@@ -488,8 +492,27 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
     std::string per_name_allocsize_name = "allocsize." + std::string(cls->tp_name);                                    \
     Stats::log(Stats::getStatId(per_name_alloc_name));                                                                 \
     Stats::log(Stats::getStatId(per_name_allocsize_name), size);
+#define ALLOC_STATS_VAR(cls)                                                                                           \
+    if (cls->tp_name) {                                                                                                \
+        std::string per_name_alloc_name = "alloc." + std::string(cls->tp_name);                                        \
+        std::string per_name_alloc_name0 = "alloc." + std::string(cls->tp_name) + "(0)";                               \
+        std::string per_name_allocsize_name = "allocsize." + std::string(cls->tp_name);                                \
+        std::string per_name_allocsize_name0 = "allocsize." + std::string(cls->tp_name) + "(0)";                       \
+        static StatCounter alloc_name(per_name_alloc_name);                                                            \
+        static StatCounter alloc_name0(per_name_alloc_name0);                                                          \
+        static StatCounter allocsize_name(per_name_allocsize_name);                                                    \
+        static StatCounter allocsize_name0(per_name_allocsize_name0);                                                  \
+        if (nitems == 0) {                                                                                             \
+            alloc_name0.log();                                                                                         \
+            allocsize_name0.log(_PyObject_VAR_SIZE(cls, nitems));                                                      \
+        } else {                                                                                                       \
+            alloc_name.log();                                                                                          \
+            allocsize_name.log(_PyObject_VAR_SIZE(cls, nitems));                                                       \
+        }                                                                                                              \
+    }
 #else
 #define ALLOC_STATS(cls)
+#define ALLOC_STATS_VAR(cls)
 #endif
 
 
@@ -525,12 +548,59 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
         /* TODO: there should be a way to not have to do this nested inlining by hand */                               \
     }
 
+#define DEFAULT_CLASS_VAR(default_cls, itemsize)                                                                       \
+    static_assert(itemsize > 0, "");                                                                                   \
+    /* asserts that the class in question is a subclass of BoxVar */                                                   \
+    inline void _base_check() {                                                                                        \
+        static_assert(std::is_base_of<BoxVar, std::remove_pointer<decltype(this)>::type>::value, "");                  \
+    }                                                                                                                  \
+                                                                                                                       \
+    void* operator new(size_t size, BoxedClass * cls, size_t nitems) __attribute__((visibility("default"))) {          \
+        assert(cls->tp_itemsize == itemsize);                                                                          \
+        return BoxVar::operator new(size, cls, nitems);                                                                \
+    }                                                                                                                  \
+    void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {                            \
+        assert(default_cls->tp_itemsize == itemsize);                                                                  \
+        return BoxVar::operator new(size, default_cls, nitems);                                                        \
+    }
+
+#define DEFAULT_CLASS_VAR_SIMPLE(default_cls, itemsize)                                                                \
+    static_assert(itemsize > 0, "");                                                                                   \
+    inline void _base_check() {                                                                                        \
+        static_assert(std::is_base_of<BoxVar, std::remove_pointer<decltype(this)>::type>::value, "");                  \
+    }                                                                                                                  \
+                                                                                                                       \
+    void* operator new(size_t size, BoxedClass * cls, size_t nitems) __attribute__((visibility("default"))) {          \
+        ALLOC_STATS_VAR(default_cls)                                                                                   \
+        assert(cls->tp_itemsize == itemsize);                                                                          \
+        return BoxVar::operator new(size, cls, nitems);                                                                \
+    }                                                                                                                  \
+    void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {                            \
+        ALLOC_STATS_VAR(default_cls)                                                                                   \
+        assert(default_cls->tp_alloc == PystonType_GenericAlloc);                                                      \
+        assert(default_cls->tp_itemsize == itemsize);                                                                  \
+        assert(default_cls->tp_basicsize == size);                                                                     \
+        assert(default_cls->is_pyston_class);                                                                          \
+        assert(default_cls->attrs_offset == 0);                                                                        \
+                                                                                                                       \
+        void* mem = gc_alloc(size + nitems * itemsize, gc::GCKind::PYTHON);                                            \
+        assert(mem);                                                                                                   \
+                                                                                                                       \
+        BoxVar* rtn = static_cast<BoxVar*>(mem);                                                                       \
+        rtn->cls = default_cls;                                                                                        \
+        rtn->ob_size = nitems;                                                                                         \
+        return rtn;                                                                                                    \
+    }
+
 // CPython C API compatibility class:
 class BoxVar : public Box {
 public:
+    // This field gets initialized in operator new.
     Py_ssize_t ob_size;
 
-    BoxVar(Py_ssize_t ob_size) : ob_size(ob_size) {}
+    BoxVar() {}
+
+    void* operator new(size_t size, BoxedClass* cls, size_t nitems) __attribute__((visibility("default")));
 };
 static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size), "");
 

src/runtime/builtin_modules/sys.cpp

@@ -461,8 +461,8 @@ void setupSys() {
     sys_module->giveAttr("maxint", boxInt(PYSTON_INT_MAX));
     sys_module->giveAttr("maxsize", boxInt(PY_SSIZE_T_MAX));
 
-    sys_flags_cls = new BoxedHeapClass(object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags), false,
-                                       static_cast<BoxedString*>(boxString("flags")));
+    sys_flags_cls = new (0) BoxedHeapClass(object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags), false,
+                                           static_cast<BoxedString*>(boxString("flags")));
     sys_flags_cls->giveAttr("__new__",
                             new BoxedFunction(boxRTFunction((void*)BoxedSysFlags::__new__, UNKNOWN, 1, 0, true, true)));
 #define ADD(name)                                                                                                      \

src/runtime/types.h

@@ -177,7 +177,10 @@ public:
     void (*simple_destructor)(Box*);
 
     // Offset of the HCAttrs object or 0 if there are no hcattrs.
+    // Negative offset is from the end of the class (useful for variable-size objects with the attrs at the end)
     // Analogous to tp_dictoffset
+    // A class should have at most of one attrs_offset and tp_dictoffset be nonzero.
+    // (But having nonzero attrs_offset here would map to having nonzero tp_dictoffset in CPython)
     const int attrs_offset;
 
     bool instancesHaveHCAttrs() { return attrs_offset != 0; }
@@ -231,12 +234,16 @@ public:
     PyBufferProcs as_buffer;
 
     BoxedString* ht_name;
-    PyObject** ht_slots;
+    PyObject* ht_slots;
+
+    typedef size_t SlotOffset;
+    SlotOffset* slotOffsets() { return (BoxedHeapClass::SlotOffset*)((char*)this + this->cls->tp_basicsize); }
+    size_t nslots() { return this->ob_size; }
 
     // These functions are the preferred way to construct new types:
     static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                   int weaklist_offset, int instance_size, bool is_user_defined, BoxedString* name,
-                                  BoxedTuple* bases);
+                                  BoxedTuple* bases, size_t nslots);
     static BoxedHeapClass* create(BoxedClass* metatype, BoxedClass* base, gcvisit_func gc_visit, int attrs_offset,
                                   int weaklist_offset, int instance_size, bool is_user_defined,
                                   const std::string& name);
@@ -251,7 +258,7 @@ private:
     friend void setupRuntime();
     friend void setupSys();
 
-    DEFAULT_CLASS(type_cls);
+    DEFAULT_CLASS_VAR(type_cls, sizeof(SlotOffset));
 };
 
 static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");
@@ -417,7 +424,7 @@ public:
     DEFAULT_CLASS_SIMPLE(bool_cls);
 };
 
-class BoxedString : public Box {
+class BoxedString : public BoxVar {
 public:
     llvm::StringRef s;
     char interned_state;
@@ -425,40 +432,28 @@ public:
     char* data() { return const_cast<char*>(s.data()); }
     size_t size() { return s.size(); }
 
-    void* operator new(size_t size, size_t ssize) __attribute__((visibility("default"))) {
-#if STAT_ALLOCATIONS
-        static StatCounter alloc_str("alloc.str");
-        static StatCounter alloc_str1("alloc.str(1)");
-        static StatCounter allocsize_str("allocsize.str");
-
-        if (ssize == 1)
-            alloc_str1.log();
-        else
-            alloc_str.log();
+    // DEFAULT_CLASS_VAR_SIMPLE doesn't work because of the +1 for the null byte
+    void* operator new(size_t size, BoxedClass* cls, size_t nitems) __attribute__((visibility("default"))) {
+        ALLOC_STATS_VAR(str_cls)
 
-        allocsize_str.log(str_cls->tp_basicsize + ssize + 1);
-#endif
-        Box* rtn = static_cast<Box*>(gc_alloc(str_cls->tp_basicsize + ssize + 1, gc::GCKind::PYTHON));
-        rtn->cls = str_cls;
-        return rtn;
+        assert(cls->tp_itemsize == sizeof(char));
+        return BoxVar::operator new(size, cls, nitems);
     }
+    void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {
+        ALLOC_STATS_VAR(str_cls)
 
-    void* operator new(size_t size, BoxedClass* cls, size_t ssize) __attribute__((visibility("default"))) {
-#if STAT_ALLOCATIONS
-        static StatCounter alloc_str("alloc.str");
-        static StatCounter alloc_str1("alloc.str(1)");
-
-        static StatCounter allocsize_str("allocsize.str");
+        assert(str_cls->tp_alloc == PystonType_GenericAlloc);
+        assert(str_cls->tp_itemsize == 1);
+        assert(str_cls->tp_basicsize == sizeof(BoxedString) + 1);
+        assert(str_cls->is_pyston_class);
+        assert(str_cls->attrs_offset == 0);
 
-        if (ssize == 1)
-            alloc_str1.log();
-        else
-            alloc_str.log();
+        void* mem = gc_alloc(sizeof(BoxedString) + 1 + nitems, gc::GCKind::PYTHON);
+        assert(mem);
 
-        allocsize_str.log(cls->tp_basicsize + ssize + 1);
-#endif
-        Box* rtn = static_cast<Box*>(cls->tp_alloc(cls, ssize + 1));
-        rtn->cls = cls;
+        BoxVar* rtn = static_cast<BoxVar*>(mem);
+        rtn->cls = str_cls;
+        rtn->ob_size = nitems;
         return rtn;
     }
 
@@ -470,7 +465,8 @@ public:
 
 private:
     // used only in ctors to give our llvm::StringRef the proper pointer
-    char* storage() { return (char*)this + cls->tp_basicsize; }
+    // Note: sizeof(BoxedString) = str_cls->tp_basicsize - 1
+    char* storage() { return (char*)this + sizeof(BoxedString); }
 
     void* operator new(size_t size) = delete;
 };
@@ -551,54 +547,13 @@ public:
     DEFAULT_CLASS_SIMPLE(list_cls);
 };
 
-class BoxedTuple : public Box {
+class BoxedTuple : public BoxVar {
 public:
     typedef std::vector<Box*, StlCompatAllocator<Box*>> GCVector;
 
     Box** elts;
 
-    void* operator new(size_t size, size_t nelts) __attribute__((visibility("default"))) {
-#if STAT_ALLOCATIONS
-        static StatCounter alloc_tuple("alloc.tuple");
-        static StatCounter alloc_tuple0("alloc.tuple(0)");
-
-        static StatCounter allocsize_tuple("allocsize.tuple");
-        static StatCounter allocsize_tuple0("allocsize.tuple(0)");
-
-        if (nelts == 0) {
-            alloc_tuple0.log();
-            allocsize_tuple0.log(_PyObject_VAR_SIZE(tuple_cls, nelts + 1));
-        } else {
-            alloc_tuple.log();
-            allocsize_tuple.log(_PyObject_VAR_SIZE(tuple_cls, nelts + 1));
-        }
-#endif
-
-        Box* rtn = static_cast<Box*>(gc_alloc(_PyObject_VAR_SIZE(tuple_cls, nelts + 1), gc::GCKind::PYTHON));
-        rtn->cls = tuple_cls;
-        return rtn;
-    }
-
-    void* operator new(size_t size, BoxedClass* cls, size_t nelts) __attribute__((visibility("default"))) {
-#if STAT_ALLOCATIONS
-        static StatCounter alloc_tuple("alloc.tuple");
-        static StatCounter alloc_tuple0("alloc.tuple(0)");
-
-        static StatCounter allocsize_tuple("allocsize.tuple");
-        static StatCounter allocsize_tuple0("allocsize.tuple(0)");
-
-        if (nelts == 0) {
-            alloc_tuple0.log();
-            allocsize_tuple0.log(_PyObject_VAR_SIZE(cls, nelts + 1));
-        } else {
-            alloc_tuple.log();
-            allocsize_tuple.log(_PyObject_VAR_SIZE(cls, nelts + 1));
-        }
-#endif
-        Box* rtn = static_cast<Box*>(cls->tp_alloc(cls, nelts));
-        rtn->cls = cls;
-        return rtn;
-    }
+    DEFAULT_CLASS_VAR_SIMPLE(tuple_cls, sizeof(Box*));
 
     static BoxedTuple* create(int64_t size) { return new (size) BoxedTuple(size); }
     static BoxedTuple* create(int64_t nelts, Box** elts) {
@@ -640,18 +595,19 @@ public:
 
     Box** begin() const { return &elts[0]; }
     Box** end() const { return &elts[nelts]; }
+    Box*& operator[](size_t index) { return elts[index]; }
 
     size_t size() const { return nelts; }
 
 private:
     size_t nelts;
 
-    BoxedTuple(size_t size) : elts(reinterpret_cast<Box**>((char*)this + this->cls->tp_basicsize)), nelts(size) {
+    BoxedTuple(size_t size) : elts(reinterpret_cast<Box**>((char*)this + tuple_cls->tp_basicsize)), nelts(size) {
         memset(elts, 0, sizeof(Box*) * size);
     }
 
     BoxedTuple(std::initializer_list<Box*>& members)
-        : elts(reinterpret_cast<Box**>((char*)this + this->cls->tp_basicsize)), nelts(members.size()) {
+        : elts(reinterpret_cast<Box**>((char*)this + tuple_cls->tp_basicsize)), nelts(members.size()) {
         // by the time we make it here elts[] is big enough to contain members
         Box** p = &elts[0];
         for (auto b : members) {

test/tests/python_slots_test.py

+print 'basic test'
+class C(object):
+    __slots__ = ['a', 'b', '__private_var']
+
+c = C()
+try:
+    print c.a
+except AttributeError as e:
+    print e.message
+c.a = 5
+print c.a
+print c.__slots__
+
+c._C__private_var = 6
+print c._C__private_var
+
+try:
+    c.x = 12
+except AttributeError as e:
+    print e.message
+
+print 'testing __dict__'
+class C(object):
+    __slots__ = ['d', 'e', '__dict__']
+c = C()
+c.d = 5
+print c.d
+c.r = 6
+print c.r
+print c.__dict__.items() # dict should contain only r (not d)
+
+print 'testing inheritance'
+class C(object):
+    __slots__ = ['a', 'b']
+class D(object):
+    __slots__ = ['c', 'd']
+class E(object):
+    pass
+
+class G(C):
+    __slots__ = ['k', 'l']
+g = G()
+g.a = 5
+print g.a
+g.k = 12
+print g.k
+
+class G(C, E):
+    __slots__ = ['k', 'l']
+g = G()
+g.a = 5
+print g.a
+g.k = 12
+print g.k
+
+class G(E, C):
+    __slots__ = ['k', 'l']
+g = G()
+g.a = 5
+print g.a
+g.k = 12
+print g.k
+
+try:
+    class G(C, D):
+        pass
+except TypeError as e:
+    print e.message
+
+print 'testing a lot of slots'
+class C(object):
+    __slots__ = ['a' + str(i) for i in xrange(1000)]
+c = C()
+c.a0 = -8
+print c.a0
+for i in xrange(1000):
+    setattr(c, 'a' + str(i), i)
+for i in xrange(1000):
+    print getattr(c, 'a' + str(i))
+
+print 'slots on a subclass of str'
+try:
+    class C(str):
+        __slots__ = ['a']
+except TypeError as e:
+    print e.message
+
+print 'slots on a class with a metaclass'
+
+class M(type):
+    def __new__(*args):
+        print "M.__new__", args[:3]
+        return type.__new__(*args)
+
+    def __call__(*args):
+        print "M.__call__", args[:3]
+        return type.__call__(*args)
+
+class C(object):
+    __metaclass__ = M
+    __slots__ = ['a', 'b']
+c = C()
+c.a = 5
+print c.a
+c.b = 6
+print c.b

test/tests/str_subclassing_gc.py

@@ -8,3 +8,8 @@ def f():
     for i in xrange(100):
         s = S(base)
 f()
+
+# make sure it has attrs
+s = S("blah")
+s.blah = 2
+print s.blah