Make listAppendInternal inlineable

- put it into a header file (and start including it)
- move the grow-the-array part into a separate function
  to encourage the fast-path to get inlined.

src/codegen/ast_interpreter.cpp

@@ -38,6 +38,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/long.h"
 #include "runtime/objmodel.h"
 #include "runtime/set.h"

src/codegen/baseline_jit.cpp

@@ -21,6 +21,7 @@
 #include "codegen/memmgr.h"
 #include "codegen/type_recording.h"
 #include "core/cfg.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"

src/codegen/runtime_hooks.cpp

@@ -40,6 +40,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/long.h"
 #include "runtime/objmodel.h"

src/runtime/builtin_modules/builtins.cpp

@@ -31,6 +31,7 @@
 #include "runtime/file.h"
 #include "runtime/ics.h"
 #include "runtime/import.h"
+#include "runtime/inline/list.h"
 #include "runtime/inline/xrange.h"
 #include "runtime/iterobject.h"
 #include "runtime/list.h"

src/runtime/builtin_modules/sys.cpp

@@ -26,6 +26,7 @@
 #include "gc/collector.h"
 #include "runtime/file.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/types.h"
 #include "runtime/util.h"

src/runtime/dict.cpp

@@ -20,6 +20,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"

src/runtime/import.cpp

@@ -24,6 +24,7 @@
 #include "codegen/parser.h"
 #include "codegen/unwinding.h"
 #include "core/ast.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 
 namespace pyston {

src/runtime/inline/link_forcer.cpp

@@ -25,6 +25,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/list.h"
 #include "runtime/long.h"

src/runtime/inline/list.cpp

@@ -13,6 +13,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "runtime/inline/list.h"
+
 #include <cstring>
 
 #include "runtime/list.h"
@@ -111,38 +113,6 @@ void BoxedList::shrink() {
     }
 }
 
-// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
-void BoxedList::ensure(int space) {
-    if (size + space > capacity) {
-        if (capacity == 0) {
-            const int INITIAL_CAPACITY = 8;
-            int initial = std::max(INITIAL_CAPACITY, space);
-            elts = new (initial) GCdArray();
-            capacity = initial;
-        } else {
-            int new_capacity = std::max(capacity * 2, size + space);
-            elts = GCdArray::realloc(elts, new_capacity);
-            capacity = new_capacity;
-        }
-    }
-    assert(capacity >= size + space);
-}
-
-// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
-extern "C" void listAppendInternal(Box* s, Box* v) {
-    // Lock must be held!
-
-    assert(isSubclass(s->cls, list_cls));
-    BoxedList* self = static_cast<BoxedList*>(s);
-
-    assert(self->size <= self->capacity);
-    self->ensure(1);
-
-    assert(self->size < self->capacity);
-    self->elts->elts[self->size] = v;
-    self->size++;
-}
-
 
 extern "C" void listAppendArrayInternal(Box* s, Box** v, int nelts) {
     // Lock must be held!

src/runtime/inline/list.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_INLINE_LIST_H
+#define PYSTON_RUNTIME_INLINE_LIST_H
+
+#include "runtime/list.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
+inline void BoxedList::grow(int min_free) {
+    if (capacity == 0) {
+        const int INITIAL_CAPACITY = 8;
+        int initial = std::max(INITIAL_CAPACITY, min_free);
+        elts = new (initial) GCdArray();
+        capacity = initial;
+    } else {
+        int new_capacity = std::max(capacity * 2, size + min_free);
+        elts = GCdArray::realloc(elts, new_capacity);
+        capacity = new_capacity;
+    }
+}
+
+inline void BoxedList::ensure(int min_free) {
+    if (unlikely(size + min_free > capacity)) {
+        grow(min_free);
+    }
+    assert(capacity >= size + min_free);
+}
+
+// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
+extern "C" inline void listAppendInternal(Box* s, Box* v) {
+    // Lock must be held!
+
+    assert(isSubclass(s->cls, list_cls));
+    BoxedList* self = static_cast<BoxedList*>(s);
+
+    assert(self->size <= self->capacity);
+    self->ensure(1);
+
+    assert(self->size < self->capacity);
+    self->elts->elts[self->size] = v;
+    self->size++;
+}
+}
+
+#endif

src/runtime/list.cpp

@@ -26,6 +26,7 @@
 #include "core/types.h"
 #include "gc/collector.h"
 #include "gc/roots.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"
@@ -33,8 +34,9 @@
 namespace pyston {
 
 extern "C" int PyList_Append(PyObject* op, PyObject* newitem) noexcept {
+    RELEASE_ASSERT(PyList_Check(op), "");
     try {
-        listAppend(op, newitem);
+        listAppendInternal(op, newitem);
     } catch (ExcInfo e) {
         abort();
     }

src/runtime/traceback.cpp

@@ -24,6 +24,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/inline/list.h"
 #include "runtime/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"

src/runtime/types.h

@@ -127,7 +127,7 @@ BoxedString* createUninitializedString(ssize_t n);
 // in theory it might work in more cases.
 char* getWriteableStringContents(BoxedString* s);
 
-extern "C" void listAppendInternal(Box* self, Box* v);
+extern "C" inline void listAppendInternal(Box* self, Box* v) __attribute__((visibility("default")));
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, Box* globals,
                               std::initializer_list<Box*> defaults) noexcept;
@@ -549,6 +549,9 @@ public:
 };
 
 class BoxedList : public Box {
+private:
+    void grow(int min_free);
+
 public:
     int64_t size, capacity;
     GCdArray* elts;
@@ -557,7 +560,7 @@ public:
 
     BoxedList() __attribute__((visibility("default"))) : size(0), capacity(0) {}
 
-    void ensure(int space);
+    void ensure(int min_free);
     void shrink();
     static const int INITIAL_CAPACITY;