Merge pull request #58 from xiafan68/delstmt

implement del x[idx], x[start, end, step]

src/analysis/function_analysis.cpp

@@ -183,6 +183,7 @@ public:
 
     virtual bool visit_assert(AST_Assert* node) { return true; }
     virtual bool visit_branch(AST_Branch* node) { return true; }
+    virtual bool visit_delete(AST_Delete* node) { return true; }
     virtual bool visit_expr(AST_Expr* node) { return true; }
     virtual bool visit_global(AST_Global* node) { return true; }
     virtual bool visit_invoke(AST_Invoke* node) { return false; }

src/analysis/type_analysis.cpp

@@ -441,6 +441,13 @@ private:
         _doSet(node->name, t);
     }
 
+    virtual void visit_delete(AST_Delete* node) {
+        for (AST_expr* target : node->targets) {
+            RELEASE_ASSERT(target->type == AST_TYPE::Subscript, "");
+            getType(target);
+        }
+    }
+
     virtual void visit_expr(AST_Expr* node) {
         if (EXPAND_UNNEEDED) {
             if (node->value != NULL)

src/codegen/irgen/irgenerator.cpp

@@ -1344,6 +1344,56 @@ private:
         cls->decvref(emitter);
     }
 
+    void doDelete(AST_Delete* node, ExcInfo exc_info) {
+        assert(state != PARTIAL);
+        for (AST_expr* target : node->targets) {
+            switch (target->type) {
+                case AST_TYPE::Subscript:
+                    _doDelitem(static_cast<AST_Subscript*>(target), exc_info);
+                    break;
+                case AST_TYPE::Attribute:
+                // delete an attribute
+                case AST_TYPE::Name:
+                    // delete a instance
+                    RELEASE_ASSERT(target->type == AST_TYPE::Subscript, "");
+                    break;
+                default:
+                    ASSERT(0, "UnSupported del target: %d", target->type);
+                    abort();
+            }
+        }
+    }
+
+    // invoke delitem in objmodel.cpp, which will invoke the listDelitem of list
+    void _doDelitem(AST_Subscript* target, ExcInfo exc_info) {
+        assert(state != PARTIAL);
+        CompilerVariable* tget = evalExpr(target->value, exc_info);
+        CompilerVariable* slice = evalExpr(target->slice, exc_info);
+
+        ConcreteCompilerVariable* converted_target = tget->makeConverted(emitter, tget->getBoxType());
+        ConcreteCompilerVariable* converted_slice = slice->makeConverted(emitter, slice->getBoxType());
+        tget->decvref(emitter);
+        slice->decvref(emitter);
+
+        bool do_patchpoint = ENABLE_ICDELITEMS && (irstate->getEffortLevel() != EffortLevel::INTERPRETED);
+        if (do_patchpoint) {
+            PatchpointSetupInfo* pp = patchpoints::createDelitemPatchpoint(emitter.currentFunction(),
+                                                                           getEmptyOpInfo(exc_info).getTypeRecorder());
+
+            std::vector<llvm::Value*> llvm_args;
+            llvm_args.push_back(converted_target->getValue());
+            llvm_args.push_back(converted_slice->getValue());
+
+            emitter.createPatchpoint(pp, (void*)pyston::delitem, llvm_args, exc_info);
+        } else {
+            emitter.getBuilder()->CreateCall2(g.funcs.delitem, converted_target->getValue(),
+                                              converted_slice->getValue());
+        }
+
+        converted_target->decvref(emitter);
+        converted_slice->decvref(emitter);
+    }
+
     CLFunction* _wrapFunction(AST_FunctionDef* node) {
         // Different compilations of the parent scope of a functiondef should lead
         // to the same CLFunction* being used:
@@ -1726,6 +1776,9 @@ private:
             case AST_TYPE::ClassDef:
                 doClassDef(ast_cast<AST_ClassDef>(node), exc_info);
                 break;
+            case AST_TYPE::Delete:
+                doDelete(ast_cast<AST_Delete>(node), exc_info);
+                break;
             case AST_TYPE::Expr:
                 doExpr(ast_cast<AST_Expr>(node), exc_info);
                 break;

src/codegen/parser.cpp

@@ -306,6 +306,16 @@ AST_Continue* read_continue(BufferedReader* reader) {
     return rtn;
 }
 
+AST_Delete* read_delete(BufferedReader* reader) {
+    AST_Delete* rtn = new AST_Delete();
+
+    rtn->col_offset = readColOffset(reader);
+    rtn->lineno = reader->readULL();
+    readExprVector(rtn->targets, reader);
+
+    return rtn;
+}
+
 AST_Dict* read_dict(BufferedReader* reader) {
     AST_Dict* rtn = new AST_Dict();
 
@@ -723,6 +733,8 @@ AST_stmt* readASTStmt(BufferedReader* reader) {
             return read_classdef(reader);
         case AST_TYPE::Continue:
             return read_continue(reader);
+        case AST_TYPE::Delete:
+            return read_delete(reader);
         case AST_TYPE::Expr:
             return read_expr(reader);
         case AST_TYPE::For:

src/codegen/patchpoints.cpp

@@ -132,6 +132,10 @@ PatchpointSetupInfo* createSetitemPatchpoint(CompiledFunction* parent_cf, TypeRe
     return PatchpointSetupInfo::initialize(true, 1, 144, parent_cf, Setitem, type_recorder);
 }
 
+PatchpointSetupInfo* createDelitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
+    return PatchpointSetupInfo::initialize(false, 1, 144, parent_cf, Delitem, type_recorder);
+}
+
 PatchpointSetupInfo* createSetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
     return PatchpointSetupInfo::initialize(false, 2, 128, parent_cf, Setattr, type_recorder);
 }

src/codegen/patchpoints.h

@@ -34,6 +34,7 @@ enum PatchpointType {
     Setattr,
     Getitem,
     Setitem,
+    Delitem,
     Binexp,
     Nonzero,
 };
@@ -92,6 +93,7 @@ PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRe
 PatchpointSetupInfo* createSetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
 PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
 PatchpointSetupInfo* createSetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
+PatchpointSetupInfo* createDelitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
 PatchpointSetupInfo* createBinexpPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
 PatchpointSetupInfo* createNonzeroPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
 }

src/codegen/runtime_hooks.cpp

@@ -162,6 +162,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(setattr);
     GET(getitem);
     GET(setitem);
+    GET(delitem);
     GET(getGlobal);
     GET(binop);
     GET(compare);

src/codegen/runtime_hooks.h

@@ -24,7 +24,8 @@ struct GlobalFuncs {
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
         *createUserClass;
     llvm::Value* getattr, *setattr, *print, *nonzero, *binop, *compare, *augbinop, *unboxedLen, *getitem, *getclsattr,
-        *getGlobal, *setitem, *unaryop, *import, *repr, *isinstance;
+        *getGlobal, *setitem, *delitem, *unaryop, *import, *repr, *isinstance;
+
     llvm::Value* checkUnpackingLength, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
         *assertNameDefined, *assertFail;
     llvm::Value* printFloat, *listAppendInternal;

src/core/ast.cpp

@@ -372,6 +372,18 @@ void AST_Continue::accept_stmt(StmtVisitor* v) {
     v->visit_continue(this);
 }
 
+void AST_Delete::accept(ASTVisitor* v) {
+    bool skip = v->visit_delete(this);
+    if (skip)
+        return;
+
+    visitVector(this->targets, v);
+}
+
+void AST_Delete::accept_stmt(StmtVisitor* v) {
+    v->visit_delete(this);
+}
+
 void AST_Dict::accept(ASTVisitor* v) {
     bool skip = v->visit_dict(this);
     if (skip)
@@ -1059,6 +1071,16 @@ bool PrintVisitor::visit_continue(AST_Continue* node) {
     return true;
 }
 
+bool PrintVisitor::visit_delete(AST_Delete* node) {
+    printf("del ");
+    for (int i = 0; i < node->targets.size(); i++) {
+        if (i > 0)
+            printf(", ");
+        node->targets[i]->accept(this);
+    }
+    return true;
+}
+
 bool PrintVisitor::visit_dict(AST_Dict* node) {
     printf("{");
     for (int i = 0; i < node->keys.size(); i++) {
@@ -1597,6 +1619,10 @@ public:
         output->push_back(node);
         return false;
     }
+    virtual bool visit_delete(AST_Delete* node) {
+        output->push_back(node);
+        return false;
+    }
     virtual bool visit_dict(AST_Dict* node) {
         output->push_back(node);
         return false;

src/core/ast.h

@@ -364,6 +364,17 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Dict;
 };
 
+class AST_Delete : public AST_stmt {
+public:
+    std::vector<AST_expr*> targets;
+    virtual void accept(ASTVisitor* v);
+    virtual void accept_stmt(StmtVisitor* v);
+
+    AST_Delete() : AST_stmt(AST_TYPE::Delete) {};
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Delete;
+};
+
 class AST_Expr : public AST_stmt {
 public:
     AST_expr* value;
@@ -862,6 +873,7 @@ public:
     virtual bool visit_comprehension(AST_comprehension* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_classdef(AST_ClassDef* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_continue(AST_Continue* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_delete(AST_Delete* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_dict(AST_Dict* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_expr(AST_Expr* node) { RELEASE_ASSERT(0, ""); }
@@ -921,6 +933,7 @@ public:
     virtual bool visit_comprehension(AST_comprehension* node) { return false; }
     virtual bool visit_classdef(AST_ClassDef* node) { return false; }
     virtual bool visit_continue(AST_Continue* node) { return false; }
+    virtual bool visit_delete(AST_Delete* node) { return false; }
     virtual bool visit_dict(AST_Dict* node) { return false; }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
@@ -997,6 +1010,7 @@ public:
     virtual void visit_augassign(AST_AugAssign* node) { RELEASE_ASSERT(0, ""); }
     virtual void visit_break(AST_Break* node) { RELEASE_ASSERT(0, ""); }
     virtual void visit_classdef(AST_ClassDef* node) { RELEASE_ASSERT(0, ""); }
+    virtual void visit_delete(AST_Delete* node) { RELEASE_ASSERT(0, ""); }
     virtual void visit_continue(AST_Continue* node) { RELEASE_ASSERT(0, ""); }
     virtual void visit_expr(AST_Expr* node) { RELEASE_ASSERT(0, ""); }
     virtual void visit_for(AST_For* node) { RELEASE_ASSERT(0, ""); }
@@ -1045,6 +1059,7 @@ public:
     virtual bool visit_classdef(AST_ClassDef* node);
     virtual bool visit_clsattribute(AST_ClsAttribute* node);
     virtual bool visit_continue(AST_Continue* node);
+    virtual bool visit_delete(AST_Delete* node);
     virtual bool visit_dict(AST_Dict* node);
     virtual bool visit_excepthandler(AST_ExceptHandler* node);
     virtual bool visit_expr(AST_Expr* node);

src/core/cfg.cpp

@@ -934,6 +934,36 @@ public:
         return true;
     }
 
+    virtual bool visit_delete(AST_Delete* node) {
+        for (auto t : node->targets) {
+            AST_Delete* astdel = new AST_Delete();
+            astdel->lineno = node->lineno;
+            astdel->col_offset = node->col_offset;
+            AST_expr* target = NULL;
+            switch (t->type) {
+                case AST_TYPE::Subscript: {
+                    AST_Subscript* s = static_cast<AST_Subscript*>(t);
+                    AST_Subscript* astsubs = new AST_Subscript();
+                    astsubs->value = remapExpr(s->value);
+                    astsubs->slice = remapExpr(s->slice);
+                    astsubs->ctx_type = AST_TYPE::Del;
+                    target = astsubs;
+                    break;
+                }
+
+                case AST_TYPE::Name:
+                case AST_TYPE::Attribute:
+                    RELEASE_ASSERT(t->type == AST_TYPE::Subscript, "");
+                default:
+                    RELEASE_ASSERT(0, "UnSupported del target: %d", t->type);
+            }
+            astdel->targets.push_back(target);
+            push_back(astdel);
+        }
+
+        return true;
+    }
+
     virtual bool visit_expr(AST_Expr* node) {
         AST_Expr* remapped = new AST_Expr();
         remapped->lineno = node->lineno;

src/core/options.cpp

@@ -38,6 +38,7 @@ bool ENABLE_ICS = 1 && _GLOBAL_ENABLE;
 bool ENABLE_ICGENERICS = 1 && ENABLE_ICS;
 bool ENABLE_ICGETITEMS = 1 && ENABLE_ICS;
 bool ENABLE_ICSETITEMS = 1 && ENABLE_ICS;
+bool ENABLE_ICDELITEMS = 1 && ENABLE_ICS;
 bool ENABLE_ICCALLSITES = 1 && ENABLE_ICS;
 bool ENABLE_ICSETATTRS = 1 && ENABLE_ICS;
 bool ENABLE_ICGETATTRS = 1 && ENABLE_ICS;

src/core/options.h

@@ -28,9 +28,10 @@ extern int MAX_OPT_ITERATIONS;
 
 extern bool SHOW_DISASM, FORCE_OPTIMIZE, BENCH, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB, ENABLE_INTERPRETER;
 
-extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICBINEXPS, ENABLE_ICNONZEROS,
-    ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENABLE_ICGETGLOBALS, ENABLE_SPECULATION, ENABLE_OSR,
-    ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES, ENABLE_TYPE_FEEDBACK;
+extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
+    ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENABLE_ICGETGLOBALS,
+    ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
+    ENABLE_TYPE_FEEDBACK;
 }
 }
 

src/runtime/inline/link_forcer.cpp

@@ -73,6 +73,7 @@ void force() {
     FORCE(getclsattr);
     FORCE(getGlobal);
     FORCE(setitem);
+    FORCE(delitem);
     FORCE(unaryop);
     FORCE(import);
     FORCE(repr);

src/runtime/inline/list.cpp

+
 // Copyright (c) 2014 Dropbox, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -52,6 +53,24 @@ Box* listiterNext(Box* s) {
     return rtn;
 }
 
+const int BoxedList::INITIAL_CAPACITY = 8;
+// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
+void BoxedList::shrink() {
+    // TODO more attention to the shrink condition to avoid frequent shrink and alloc
+    if (capacity > size * 3) {
+        int new_capacity = std::max(static_cast<int64_t>(INITIAL_CAPACITY), capacity / 2);
+        if (size > 0) {
+            elts = (BoxedList::ElementArray*)rt_realloc(elts,
+                                                        new_capacity * sizeof(Box*) + sizeof(BoxedList::ElementArray));
+            capacity = new_capacity;
+        } else if (size == 0) {
+            rt_free(elts);
+            elts = NULL;
+            capacity = 0;
+        }
+    }
+}
+
 // TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
 void BoxedList::ensure(int space) {
     if (size + space > capacity) {

src/runtime/list.cpp

@@ -205,6 +205,49 @@ extern "C" Box* listSetitem(BoxedList* self, Box* slice, Box* v) {
     }
 }
 
+extern "C" Box* listDelitemInt(BoxedList* self, BoxedInt* slice) {
+    int64_t n = slice->n;
+    if (n < 0)
+        n = self->size + n;
+
+    if (n < 0 || n >= self->size) {
+        raiseExcHelper(IndexError, "list index out of range");
+    }
+    memmove(self->elts->elts + n, self->elts->elts + n + 1, (self->size - n - 1) * sizeof(Box*));
+    self->size--;
+    return None;
+}
+
+extern "C" Box* listDelitemSlice(BoxedList* self, BoxedSlice* slice) {
+    i64 start, stop, step;
+    parseSlice(slice, self->size, &start, &stop, &step);
+    RELEASE_ASSERT(step == 1, "step sizes must be 1 for now");
+
+    assert(0 <= start && start < self->size);
+    ASSERT(0 <= stop && stop <= self->size, "%ld %ld", self->size, stop);
+    assert(start <= stop);
+
+    int remaining_elts = self->size - stop;
+
+    memmove(self->elts->elts + start, self->elts->elts + stop, remaining_elts * sizeof(Box*));
+    self->size -= (stop - start);
+    return None;
+}
+
+extern "C" Box* listDelitem(BoxedList* self, Box* slice) {
+    Box* rtn;
+
+    if (slice->cls == int_cls) {
+        rtn = listDelitemInt(self, static_cast<BoxedInt*>(slice));
+    } else if (slice->cls == slice_cls) {
+        rtn = listDelitemSlice(self, static_cast<BoxedSlice*>(slice));
+    } else {
+        raiseExcHelper(TypeError, "list indices must be integers, not %s", getTypeName(slice)->c_str());
+    }
+    self->shrink();
+    return rtn;
+}
+
 extern "C" Box* listInsert(BoxedList* self, Box* idx, Box* v) {
     if (idx->cls != int_cls) {
         raiseExcHelper(TypeError, "an integer is required");
@@ -412,6 +455,12 @@ void setupList() {
     addRTFunction(setitem, (void*)listSetitem, NULL, std::vector<ConcreteCompilerType*>{ LIST, NULL, NULL }, false);
     list_cls->giveAttr("__setitem__", new BoxedFunction(setitem));
 
+    CLFunction* delitem = createRTFunction();
+    addRTFunction(delitem, (void*)listDelitemInt, NULL, std::vector<ConcreteCompilerType*>{ LIST, BOXED_INT }, false);
+    addRTFunction(delitem, (void*)listDelitemSlice, NULL, std::vector<ConcreteCompilerType*>{ LIST, SLICE }, false);
+    addRTFunction(delitem, (void*)listDelitem, NULL, std::vector<ConcreteCompilerType*>{ LIST, NULL }, false);
+    list_cls->giveAttr("__delitem__", new BoxedFunction(delitem));
+
     list_cls->giveAttr("insert", new BoxedFunction(boxRTFunction((void*)listInsert, NULL, 3, false)));
     list_cls->giveAttr("__mul__", new BoxedFunction(boxRTFunction((void*)listMul, NULL, 2, false)));
 

src/runtime/objmodel.cpp

@@ -2140,6 +2140,40 @@ extern "C" void setitem(Box* target, Box* slice, Box* value) {
     }
 }
 
+// del target[start:end:step]
+extern "C" void delitem(Box* target, Box* slice) {
+    static StatCounter slowpath_delitem("slowpath_delitem");
+    slowpath_delitem.log();
+    static std::string str_delitem("__delitem__");
+
+    std::unique_ptr<Rewriter> rewriter(
+        Rewriter::createRewriter(__builtin_extract_return_addr(__builtin_return_address(0)), 2, 1, "delitem"));
+
+    Box* rtn;
+    RewriterVar r_rtn;
+    if (rewriter.get()) {
+        CallRewriteArgs rewrite_args(rewriter.get(), rewriter->getArg(0));
+        rewrite_args.arg1 = rewriter->getArg(1);
+
+        rtn = callattrInternal1(target, &str_delitem, CLASS_ONLY, &rewrite_args, 1, slice);
+
+        if (!rewrite_args.out_success)
+            rewriter.reset(NULL);
+        else if (rtn)
+            r_rtn = rewrite_args.out_rtn;
+    } else {
+        rtn = callattrInternal1(target, &str_delitem, CLASS_ONLY, NULL, 1, slice);
+    }
+
+    if (rtn == NULL) {
+        raiseExcHelper(TypeError, "'%s' object does not support item deletion", getTypeName(target)->c_str());
+    }
+
+    if (rewriter.get()) {
+        rewriter->commit();
+    }
+}
+
 // A wrapper around the HCBox constructor
 // TODO is there a way to avoid the indirection?
 static Box* makeHCBox(ObjectFlavor* flavor, BoxedClass* cls) {

src/runtime/objmodel.h

@@ -62,6 +62,7 @@ extern "C" Box* augbinop(Box* lhs, Box* rhs, int op_type);
 extern "C" Box* getGlobal(BoxedModule* m, std::string* name);
 extern "C" Box* getitem(Box* value, Box* slice);
 extern "C" void setitem(Box* target, Box* slice, Box* value);
+extern "C" void delitem(Box* target, Box* slice);
 extern "C" Box* getclsattr(Box* obj, const char* attr);
 extern "C" Box* unaryop(Box* operand, int op_type);
 extern "C" Box* import(const std::string* name);

src/runtime/types.h

@@ -219,6 +219,8 @@ public:
     BoxedList() __attribute__((visibility("default"))) : Box(&list_flavor, list_cls), size(0), capacity(0) {}
 
     void ensure(int space);
+    void shrink();
+    static const int INITIAL_CAPACITY;
 };
 
 class BoxedTuple : public Box {

test/tests/delitem.py

+a=[i for i in range(10)]
+del a[0]
+print a
+del a[-1]
+print a
+del a[1]
+print a
+
+del a[0:2] 
+print a 
+del a[1:3:1]
+print a
+#test del all
+del a[:]
+print a
+a.append(1)
+print a