Commit 7cd29d79 authored by Chris Toshok's avatar Chris Toshok

mucho cleanup

parent 1a268096
...@@ -35,15 +35,40 @@ namespace gc { ...@@ -35,15 +35,40 @@ namespace gc {
void _doFree(GCAllocation* al); void _doFree(GCAllocation* al);
// these template functions are for both large and huge sections // lots of linked lists around here, so let's just use template functions for them all
template <class ListT> inline void unlinkNode(ListT* node) { template <class ListT> inline void nullNextPrev(ListT* node) {
node->next = NULL;
node->prev = NULL;
}
template <class ListT> inline void removeFromLL(ListT* node) {
*node->prev = node->next;
if (node->next)
node->next->prev = node->prev;
}
template <class ListT> inline void removeFromLLAndNull(ListT* node) {
*node->prev = node->next; *node->prev = node->next;
if (node->next) if (node->next)
node->next->prev = node->prev; node->next->prev = node->prev;
nullNextPrev(node);
}
template <class ListT> inline void insertIntoLL(ListT** next_pointer, ListT* next) {
assert(next_pointer);
assert(next);
assert(!next->next);
assert(!next->prev);
next->next = *next_pointer;
if (next->next)
next->next->prev = &next->next;
*next_pointer = next;
next->prev = next_pointer;
} }
template <class ListT, typename Free>
inline void sweepHeap(ListT* head, std::function<void(GCAllocation*)> __free, Free free_func) { template <class ListT, typename Free> inline void sweepList(ListT* head, Free free_func) {
auto cur = head; auto cur = head;
while (cur) { while (cur) {
GCAllocation* al = cur->data; GCAllocation* al = cur->data;
...@@ -51,9 +76,9 @@ inline void sweepHeap(ListT* head, std::function<void(GCAllocation*)> __free, Fr ...@@ -51,9 +76,9 @@ inline void sweepHeap(ListT* head, std::function<void(GCAllocation*)> __free, Fr
clearMark(al); clearMark(al);
cur = cur->next; cur = cur->next;
} else { } else {
__free(al); _doFree(al);
unlinkNode(cur); removeFromLL(cur);
auto to_free = cur; auto to_free = cur;
cur = cur->next; cur = cur->next;
...@@ -92,6 +117,121 @@ void registerGCManagedBytes(size_t bytes) { ...@@ -92,6 +117,121 @@ void registerGCManagedBytes(size_t bytes) {
Heap global_heap; Heap global_heap;
void _doFree(GCAllocation* al) {
if (VERBOSITY() >= 2)
printf("Freeing %p\n", al->user_data);
#ifndef NVALGRIND
VALGRIND_DISABLE_ERROR_REPORTING;
#endif
GCKind alloc_kind = al->kind_id;
#ifndef NVALGRIND
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
if (alloc_kind == GCKind::PYTHON) {
#ifndef NVALGRIND
VALGRIND_DISABLE_ERROR_REPORTING;
#endif
Box* b = (Box*)al->user_data;
#ifndef NVALGRIND
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
ASSERT(b->cls->tp_dealloc == NULL, "%s", getTypeName(b));
if (b->cls->simple_destructor)
b->cls->simple_destructor(b);
}
}
void Heap::destructContents(GCAllocation* al) {
_doFree(al);
}
struct HeapStatistics {
struct TypeStats {
int64_t nallocs;
int64_t nbytes;
TypeStats() : nallocs(0), nbytes(0) {}
void print(const char* name) const {
if (nbytes > (1 << 20))
printf("%s: %ld allocations for %.1f MB\n", name, nallocs, nbytes * 1.0 / (1 << 20));
else if (nbytes > (1 << 10))
printf("%s: %ld allocations for %.1f KB\n", name, nallocs, nbytes * 1.0 / (1 << 10));
else
printf("%s: %ld allocations for %ld bytes\n", name, nallocs, nbytes);
}
};
std::unordered_map<BoxedClass*, TypeStats> by_cls;
TypeStats conservative, untracked;
TypeStats total;
};
void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
stats->total.nallocs++;
stats->total.nbytes += nbytes;
if (al->kind_id == GCKind::PYTHON) {
Box* b = (Box*)al->user_data;
auto& t = stats->by_cls[b->cls];
t.nallocs++;
t.nbytes += nbytes;
} else if (al->kind_id == GCKind::CONSERVATIVE) {
stats->conservative.nallocs++;
stats->conservative.nbytes += nbytes;
} else if (al->kind_id == GCKind::UNTRACKED) {
stats->untracked.nallocs++;
stats->untracked.nbytes += nbytes;
} else {
RELEASE_ASSERT(0, "%d", (int)al->kind_id);
}
}
void Heap::dumpHeapStatistics() {
threading::GLPromoteRegion _lock;
HeapStatistics stats;
small_arena.getStatistics(&stats);
large_arena.getStatistics(&stats);
huge_arena.getStatistics(&stats);
stats.conservative.print("conservative");
stats.untracked.print("untracked");
for (const auto& p : stats.by_cls) {
p.second.print(getFullNameOfClass(p.first).c_str());
}
stats.total.print("Total");
printf("\n");
}
void dumpHeapStatistics() {
global_heap.dumpHeapStatistics();
}
//////
/// Small Arena
GCAllocation* SmallArena::alloc(size_t bytes) {
registerGCManagedBytes(bytes);
if (bytes <= 16)
return _alloc(16, 0);
else if (bytes <= 32)
return _alloc(32, 1);
else {
for (int i = 2; i < NUM_BUCKETS; i++) {
if (sizes[i] >= bytes) {
return _alloc(sizes[i], i);
}
}
return NULL;
}
}
GCAllocation* SmallArena::realloc(GCAllocation* al, size_t bytes) { GCAllocation* SmallArena::realloc(GCAllocation* al, size_t bytes) {
Block* b = Block::forPointer(al); Block* b = Block::forPointer(al);
...@@ -110,10 +250,25 @@ GCAllocation* SmallArena::realloc(GCAllocation* al, size_t bytes) { ...@@ -110,10 +250,25 @@ GCAllocation* SmallArena::realloc(GCAllocation* al, size_t bytes) {
memcpy(rtn, al, std::min(bytes, size)); memcpy(rtn, al, std::min(bytes, size));
#endif #endif
_free(al, b); free(al);
return rtn; return rtn;
} }
void SmallArena::free(GCAllocation* alloc) {
Block* b = Block::forPointer(alloc);
size_t size = b->size;
int offset = (char*)alloc - (char*)b;
assert(offset % size == 0);
int atom_idx = offset / ATOM_SIZE;
assert(!b->isfree.isSet(atom_idx));
b->isfree.set(atom_idx);
#ifndef NVALGRIND
// VALGRIND_MEMPOOL_FREE(b, ptr);
#endif
}
GCAllocation* SmallArena::allocationFrom(void* ptr) { GCAllocation* SmallArena::allocationFrom(void* ptr) {
Block* b = Block::forPointer(ptr); Block* b = Block::forPointer(ptr);
size_t size = b->size; size_t size = b->size;
...@@ -131,37 +286,6 @@ GCAllocation* SmallArena::allocationFrom(void* ptr) { ...@@ -131,37 +286,6 @@ GCAllocation* SmallArena::allocationFrom(void* ptr) {
return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]); return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]);
} }
SmallArena::Block** SmallArena::freeChain(Block** head) {
while (Block* b = *head) {
int num_objects = b->numObjects();
int first_obj = b->minObjIndex();
int atoms_per_obj = b->atomsPerObj();
for (int obj_idx = first_obj; obj_idx < num_objects; obj_idx++) {
int atom_idx = obj_idx * atoms_per_obj;
if (b->isfree.isSet(atom_idx))
continue;
void* p = &b->atoms[atom_idx];
GCAllocation* al = reinterpret_cast<GCAllocation*>(p);
if (isMarked(al)) {
clearMark(al);
} else {
_doFree(al);
// assert(p != (void*)0x127000d960); // the main module
b->isfree.set(atom_idx);
}
}
head = &b->next;
}
return head;
}
void SmallArena::freeUnmarked() { void SmallArena::freeUnmarked() {
thread_caches.forEachValue([this](ThreadBlockCache* cache) { thread_caches.forEachValue([this](ThreadBlockCache* cache) {
for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) { for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
...@@ -179,185 +303,227 @@ void SmallArena::freeUnmarked() { ...@@ -179,185 +303,227 @@ void SmallArena::freeUnmarked() {
break; break;
} }
if (h) { if (h) {
removeFromLL(h); removeFromLLAndNull(h);
insertIntoLL(&heads[bidx], h); insertIntoLL(&heads[bidx], h);
} }
Block** chain_end = freeChain(&cache->cache_free_heads[bidx]); Block** chain_end = _freeChain(&cache->cache_free_heads[bidx]);
freeChain(&cache->cache_full_heads[bidx]); _freeChain(&cache->cache_full_heads[bidx]);
while (Block* b = cache->cache_full_heads[bidx]) { while (Block* b = cache->cache_full_heads[bidx]) {
removeFromLL(b); removeFromLLAndNull(b);
insertIntoLL(chain_end, b); insertIntoLL(chain_end, b);
} }
} }
}); });
for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) { for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
Block** chain_end = freeChain(&heads[bidx]); Block** chain_end = _freeChain(&heads[bidx]);
freeChain(&full_heads[bidx]); _freeChain(&full_heads[bidx]);
while (Block* b = full_heads[bidx]) { while (Block* b = full_heads[bidx]) {
removeFromLL(b); removeFromLLAndNull(b);
insertIntoLL(chain_end, b); insertIntoLL(chain_end, b);
} }
} }
} }
// TODO: copy-pasted from freeUnmarked()
void SmallArena::getStatistics(HeapStatistics* stats) {
thread_caches.forEachValue([this, stats](ThreadBlockCache* cache) {
for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
Block* h = cache->cache_free_heads[bidx];
#define LARGE_BLOCK_NUM_CHUNKS ((BLOCK_SIZE >> CHUNK_BITS) - 1) _getChainStatistics(stats, &cache->cache_free_heads[bidx]);
_getChainStatistics(stats, &cache->cache_full_heads[bidx]);
}
});
#define LARGE_BLOCK_FOR_OBJ(obj) ((LargeBlock*)((int64_t)(obj) & ~(int64_t)(BLOCK_SIZE - 1))) for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
#define LARGE_CHUNK_INDEX(obj, section) (((char*)(obj) - (char*)(section)) >> CHUNK_BITS) _getChainStatistics(stats, &heads[bidx]);
_getChainStatistics(stats, &full_heads[bidx]);
}
}
int64_t los_memory_usage = 0;
static int64_t large_object_count = 0; SmallArena::Block** SmallArena::_freeChain(Block** head) {
static int large_block_count = 0; while (Block* b = *head) {
int num_objects = b->numObjects();
int first_obj = b->minObjIndex();
int atoms_per_obj = b->atomsPerObj();
void LargeArena::add_free_chunk(LargeFreeChunk* free_chunks, size_t size) { for (int obj_idx = first_obj; obj_idx < num_objects; obj_idx++) {
size_t num_chunks = size >> CHUNK_BITS; int atom_idx = obj_idx * atoms_per_obj;
free_chunks->size = size; if (b->isfree.isSet(atom_idx))
continue;
if (num_chunks >= NUM_FREE_LISTS) void* p = &b->atoms[atom_idx];
num_chunks = 0; GCAllocation* al = reinterpret_cast<GCAllocation*>(p);
free_chunks->next_size = free_lists[num_chunks];
free_lists[num_chunks] = free_chunks;
}
LargeArena::LargeFreeChunk* LargeArena::get_from_size_list(LargeFreeChunk** list, size_t size) { if (isMarked(al)) {
LargeFreeChunk* free_chunks = NULL; clearMark(al);
LargeBlock* section; } else {
size_t i, num_chunks, start_index; _doFree(al);
assert((size & (CHUNK_SIZE - 1)) == 0); // assert(p != (void*)0x127000d960); // the main module
b->isfree.set(atom_idx);
}
}
while (*list) { head = &b->next;
free_chunks = *list;
if (free_chunks->size >= size)
break;
list = &(*list)->next_size;
} }
return head;
}
if (!*list)
return NULL;
*list = free_chunks->next_size; SmallArena::Block* SmallArena::_allocBlock(uint64_t size, Block** prev) {
Block* rtn = (Block*)doMmap(sizeof(Block));
assert(rtn);
rtn->size = size;
rtn->num_obj = BLOCK_SIZE / size;
rtn->min_obj_index = (BLOCK_HEADER_SIZE + size - 1) / size;
rtn->atoms_per_obj = size / ATOM_SIZE;
rtn->prev = prev;
rtn->next = NULL;
if (free_chunks->size > size) #ifndef NVALGRIND
add_free_chunk((LargeFreeChunk*)((char*)free_chunks + size), free_chunks->size - size); // Not sure if this mempool stuff is better than the malloc-like interface:
// VALGRIND_CREATE_MEMPOOL(rtn, 0, true);
#endif
num_chunks = size >> CHUNK_BITS; // printf("Allocated new block %p\n", rtn);
section = LARGE_BLOCK_FOR_OBJ(free_chunks); // Don't think I need to do this:
rtn->isfree.setAllZero();
rtn->next_to_check.reset();
start_index = LARGE_CHUNK_INDEX(free_chunks, section); int num_objects = rtn->numObjects();
for (i = start_index; i < start_index + num_chunks; ++i) { int num_lost = rtn->minObjIndex();
assert(section->free_chunk_map[i]); int atoms_per_object = rtn->atomsPerObj();
section->free_chunk_map[i] = 0; for (int i = num_lost * atoms_per_object; i < num_objects * atoms_per_object; i += atoms_per_object) {
rtn->isfree.set(i);
// printf("%d %d\n", idx, bit);
} }
section->num_free_chunks -= size >> CHUNK_BITS; // printf("%d %d %d\n", num_objects, num_lost, atoms_per_object);
assert(section->num_free_chunks >= 0); // for (int i =0; i < BITFIELD_ELTS; i++) {
// printf("%d: %lx\n", i, rtn->isfree[i]);
return free_chunks; //}
} return rtn;
}
LargeArena::LargeObj* LargeArena::_allocInternal(size_t size) {
LargeBlock* section;
LargeFreeChunk* free_chunks;
size_t num_chunks;
size += CHUNK_SIZE - 1;
size &= ~(CHUNK_SIZE - 1);
num_chunks = size >> CHUNK_BITS; SmallArena::ThreadBlockCache::~ThreadBlockCache() {
LOCK_REGION(heap->lock);
assert(size > 0 && size - sizeof(LargeObj) <= ALLOC_SIZE_LIMIT); for (int i = 0; i < NUM_BUCKETS; i++) {
assert(num_chunks > 0); while (Block* b = cache_free_heads[i]) {
removeFromLLAndNull(b);
insertIntoLL(&small->heads[i], b);
}
retry: while (Block* b = cache_full_heads[i]) {
if (num_chunks >= NUM_FREE_LISTS) { removeFromLLAndNull(b);
free_chunks = get_from_size_list(&free_lists[0], size); insertIntoLL(&small->full_heads[i], b);
} else {
size_t i;
for (i = num_chunks; i < NUM_FREE_LISTS; ++i) {
free_chunks = get_from_size_list(&free_lists[i], size);
if (free_chunks)
break;
} }
if (!free_chunks)
free_chunks = get_from_size_list(&free_lists[0], size);
} }
}
if (free_chunks) GCAllocation* SmallArena::_allocFromBlock(Block* b) {
return (LargeObj*)free_chunks; int idx = b->isfree.scanForNext(b->next_to_check);
if (idx == -1)
return NULL;
section = (LargeBlock*)doMmap(BLOCK_SIZE); void* rtn = &b->atoms[idx];
return reinterpret_cast<GCAllocation*>(rtn);
}
if (!section) SmallArena::Block* SmallArena::_claimBlock(size_t rounded_size, Block** free_head) {
return NULL; Block* free_block = *free_head;
if (free_block) {
removeFromLLAndNull(free_block);
return free_block;
}
free_chunks = (LargeFreeChunk*)((char*)section + CHUNK_SIZE); return _allocBlock(rounded_size, NULL);
free_chunks->size = BLOCK_SIZE - CHUNK_SIZE; }
free_chunks->next_size = free_lists[0];
free_lists[0] = free_chunks;
section->num_free_chunks = LARGE_BLOCK_NUM_CHUNKS; GCAllocation* SmallArena::_alloc(size_t rounded_size, int bucket_idx) {
Block** free_head = &heads[bucket_idx];
Block** full_head = &full_heads[bucket_idx];
section->free_chunk_map = (unsigned char*)section + sizeof(LargeBlock); ThreadBlockCache* cache = thread_caches.get();
assert(sizeof(LargeBlock) + LARGE_BLOCK_NUM_CHUNKS + 1 <= CHUNK_SIZE);
section->free_chunk_map[0] = 0;
memset(section->free_chunk_map + 1, 1, LARGE_BLOCK_NUM_CHUNKS);
section->next = blocks; Block** cache_head = &cache->cache_free_heads[bucket_idx];
blocks = section;
++large_block_count; // static __thread int gc_allocs = 0;
// if (++gc_allocs == 128) {
// static StatCounter sc_total("gc_allocs");
// sc_total.log(128);
// gc_allocs = 0;
//}
goto retry; while (true) {
} while (Block* cache_block = *cache_head) {
GCAllocation* rtn = _allocFromBlock(cache_block);
if (rtn)
return rtn;
void LargeArena::_freeInternal(LargeObj* obj, size_t size) { removeFromLLAndNull(cache_block);
LargeBlock* section = LARGE_BLOCK_FOR_OBJ(obj); insertIntoLL(&cache->cache_full_heads[bucket_idx], cache_block);
size_t num_chunks, i, start_index; }
size += CHUNK_SIZE - 1; // Not very useful to count the cache misses if we don't count the total attempts:
size &= ~(CHUNK_SIZE - 1); // static StatCounter sc_fallback("gc_allocs_cachemiss");
// sc_fallback.log();
num_chunks = size >> CHUNK_BITS; LOCK_REGION(heap->lock);
assert(size > 0 && size - sizeof(LargeObj) <= ALLOC_SIZE_LIMIT); assert(*cache_head == NULL);
assert(num_chunks > 0);
section->num_free_chunks += num_chunks; // should probably be called allocBlock:
assert(section->num_free_chunks <= LARGE_BLOCK_NUM_CHUNKS); Block* myblock = _claimBlock(rounded_size, &heads[bucket_idx]);
assert(myblock);
assert(!myblock->next);
assert(!myblock->prev);
/* // printf("%d claimed new block %p with %d objects\n", threading::gettid(), myblock, myblock->numObjects());
* We could free the LOS section here if it's empty, but we
* can't unless we also remove its free chunks from the fast
* free lists. Instead, we do it in los_sweep().
*/
start_index = LARGE_CHUNK_INDEX(obj, section); insertIntoLL(cache_head, myblock);
for (i = start_index; i < start_index + num_chunks; ++i) {
assert(!section->free_chunk_map[i]);
section->free_chunk_map[i] = 1;
} }
add_free_chunk((LargeFreeChunk*)obj, size);
} }
void LargeArena::_free(LargeObj* obj) { // TODO: copy-pasted from _freeChain
unlinkNode(obj); void SmallArena::_getChainStatistics(HeapStatistics* stats, Block** head) {
_freeInternal(obj, obj->size); while (Block* b = *head) {
} int num_objects = b->numObjects();
int first_obj = b->minObjIndex();
int atoms_per_obj = b->atomsPerObj();
void LargeArena::freeUnmarked() { for (int obj_idx = first_obj; obj_idx < num_objects; obj_idx++) {
sweepHeap(head, _doFree, [this](LargeObj* ptr) { _freeInternal(ptr, ptr->size); }); int atom_idx = obj_idx * atoms_per_obj;
if (b->isfree.isSet(atom_idx))
continue;
void* p = &b->atoms[atom_idx];
GCAllocation* al = reinterpret_cast<GCAllocation*>(p);
addStatistic(stats, al, b->size);
}
head = &b->next;
}
} }
//////
/// Large Arena
#define LARGE_BLOCK_NUM_CHUNKS ((BLOCK_SIZE >> CHUNK_BITS) - 1)
#define LARGE_BLOCK_FOR_OBJ(obj) ((LargeBlock*)((int64_t)(obj) & ~(int64_t)(BLOCK_SIZE - 1)))
#define LARGE_CHUNK_INDEX(obj, section) (((char*)(obj) - (char*)(section)) >> CHUNK_BITS)
GCAllocation* LargeArena::alloc(size_t size) { GCAllocation* LargeArena::alloc(size_t size) {
registerGCManagedBytes(size); registerGCManagedBytes(size);
...@@ -365,22 +531,18 @@ GCAllocation* LargeArena::alloc(size_t size) { ...@@ -365,22 +531,18 @@ GCAllocation* LargeArena::alloc(size_t size) {
// printf ("allocLarge %zu\n", size); // printf ("allocLarge %zu\n", size);
LargeObj* obj = _allocInternal(size + sizeof(GCAllocation) + sizeof(LargeObj)); LargeObj* obj = _alloc(size + sizeof(GCAllocation) + sizeof(LargeObj));
obj->size = size; obj->size = size;
obj->next = head; nullNextPrev(obj);
if (obj->next) insertIntoLL(&head, obj);
obj->next->prev = &obj->next;
obj->prev = &head;
head = obj;
large_object_count++;
return obj->data; return obj->data;
} }
GCAllocation* LargeArena::realloc(GCAllocation* al, size_t bytes) { GCAllocation* LargeArena::realloc(GCAllocation* al, size_t bytes) {
LargeObj* obj = (LargeObj*)((char*)al - offsetof(LargeObj, data)); LargeObj* obj = LargeObj::fromAllocation(al);
int size = obj->size; int size = obj->size;
if (size >= bytes && size < bytes * 2) if (size >= bytes && size < bytes * 2)
return al; return al;
...@@ -388,13 +550,12 @@ GCAllocation* LargeArena::realloc(GCAllocation* al, size_t bytes) { ...@@ -388,13 +550,12 @@ GCAllocation* LargeArena::realloc(GCAllocation* al, size_t bytes) {
GCAllocation* rtn = heap->alloc(bytes); GCAllocation* rtn = heap->alloc(bytes);
memcpy(rtn, al, std::min(bytes, obj->size)); memcpy(rtn, al, std::min(bytes, obj->size));
_free(obj); _freeLargeObj(obj);
return rtn; return rtn;
} }
void LargeArena::free(GCAllocation* al) { void LargeArena::free(GCAllocation* al) {
LargeObj* obj = (LargeObj*)((char*)al - offsetof(LargeObj, data)); _freeLargeObj(LargeObj::fromAllocation(al));
_free(obj);
} }
GCAllocation* LargeArena::allocationFrom(void* ptr) { GCAllocation* LargeArena::allocationFrom(void* ptr) {
...@@ -410,341 +571,206 @@ GCAllocation* LargeArena::allocationFrom(void* ptr) { ...@@ -410,341 +571,206 @@ GCAllocation* LargeArena::allocationFrom(void* ptr) {
return NULL; return NULL;
} }
void HugeArena::freeUnmarked() { void LargeArena::freeUnmarked() {
sweepHeap(head, _doFree, [this](HugeObj* ptr) { _freeHugeObj(ptr); }); sweepList(head, [this](LargeObj* ptr) { _freeLargeObj(ptr); });
}
GCAllocation* HugeArena::alloc(size_t size) {
registerGCManagedBytes(size);
LOCK_REGION(heap->lock);
size_t total_size = size + sizeof(HugeObj);
total_size = (total_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
HugeObj* rtn = (HugeObj*)doMmap(total_size);
rtn->obj_size = size;
rtn->next = head;
if (rtn->next)
rtn->next->prev = &rtn->next;
rtn->prev = &head;
head = rtn;
return rtn->data;
}
GCAllocation* HugeArena::realloc(GCAllocation* al, size_t bytes) {
HugeObj* lobj = HugeObj::fromAllocation(al);
int capacity = lobj->capacity();
if (capacity >= bytes && capacity < bytes * 2)
return al;
GCAllocation* rtn = heap->alloc(bytes);
memcpy(rtn, al, std::min(bytes, lobj->obj_size));
_freeHugeObj(lobj);
return rtn;
}
void HugeArena::_freeHugeObj(HugeObj* lobj) {
unlinkNode(lobj);
int r = munmap(lobj, lobj->mmap_size());
assert(r == 0);
}
void HugeArena::free(GCAllocation* al) {
HugeObj* lobj = HugeObj::fromAllocation(al);
_freeHugeObj(lobj);
} }
GCAllocation* HugeArena::allocationFrom(void* ptr) { void LargeArena::getStatistics(HeapStatistics* stats) {
HugeObj* cur = head; LargeObj* cur = head;
while (cur) { while (cur) {
if (ptr >= cur && ptr < &cur->data[cur->obj_size]) GCAllocation* al = cur->data;
return &cur->data[0]; addStatistic(stats, al, cur->size);
cur = cur->next; cur = cur->next;
} }
return NULL;
} }
SmallArena::Block* SmallArena::alloc_block(uint64_t size, Block** prev) {
Block* rtn = (Block*)doMmap(sizeof(Block));
assert(rtn);
rtn->size = size;
rtn->num_obj = BLOCK_SIZE / size;
rtn->min_obj_index = (BLOCK_HEADER_SIZE + size - 1) / size;
rtn->atoms_per_obj = size / ATOM_SIZE;
rtn->prev = prev;
rtn->next = NULL;
#ifndef NVALGRIND
// Not sure if this mempool stuff is better than the malloc-like interface:
// VALGRIND_CREATE_MEMPOOL(rtn, 0, true);
#endif
// printf("Allocated new block %p\n", rtn); void LargeArena::add_free_chunk(LargeFreeChunk* free_chunks, size_t size) {
size_t num_chunks = size >> CHUNK_BITS;
// Don't think I need to do this:
rtn->isfree.setAllZero();
rtn->next_to_check.reset();
int num_objects = rtn->numObjects(); free_chunks->size = size;
int num_lost = rtn->minObjIndex();
int atoms_per_object = rtn->atomsPerObj();
for (int i = num_lost * atoms_per_object; i < num_objects * atoms_per_object; i += atoms_per_object) {
rtn->isfree.set(i);
// printf("%d %d\n", idx, bit);
}
// printf("%d %d %d\n", num_objects, num_lost, atoms_per_object); if (num_chunks >= NUM_FREE_LISTS)
// for (int i =0; i < BITFIELD_ELTS; i++) { num_chunks = 0;
// printf("%d: %lx\n", i, rtn->isfree[i]); free_chunks->next_size = free_lists[num_chunks];
//} free_lists[num_chunks] = free_chunks;
return rtn;
} }
void SmallArena::insertIntoLL(Block** next_pointer, Block* next) { LargeArena::LargeFreeChunk* LargeArena::get_from_size_list(LargeFreeChunk** list, size_t size) {
assert(next_pointer); LargeFreeChunk* free_chunks = NULL;
assert(next); LargeBlock* section;
assert(!next->next); size_t i, num_chunks, start_index;
assert(!next->prev);
next->next = *next_pointer; assert((size & (CHUNK_SIZE - 1)) == 0);
if (next->next)
next->next->prev = &next->next;
*next_pointer = next;
next->prev = next_pointer;
}
void SmallArena::removeFromLL(Block* b) { while (*list) {
unlinkNode(b); free_chunks = *list;
b->next = NULL; if (free_chunks->size >= size)
b->prev = NULL; break;
} list = &(*list)->next_size;
}
SmallArena::ThreadBlockCache::~ThreadBlockCache() { if (!*list)
LOCK_REGION(heap->lock); return NULL;
for (int i = 0; i < NUM_BUCKETS; i++) { *list = free_chunks->next_size;
while (Block* b = cache_free_heads[i]) {
small->removeFromLL(b);
small->insertIntoLL(&small->heads[i], b);
}
while (Block* b = cache_full_heads[i]) { if (free_chunks->size > size)
small->removeFromLL(b); add_free_chunk((LargeFreeChunk*)((char*)free_chunks + size), free_chunks->size - size);
small->insertIntoLL(&small->full_heads[i], b);
}
}
}
GCAllocation* SmallArena::allocFromBlock(Block* b) { num_chunks = size >> CHUNK_BITS;
int idx = b->isfree.scanForNext(b->next_to_check);
if (idx == -1)
return NULL;
void* rtn = &b->atoms[idx]; section = LARGE_BLOCK_FOR_OBJ(free_chunks);
return reinterpret_cast<GCAllocation*>(rtn);
}
SmallArena::Block* SmallArena::claimBlock(size_t rounded_size, Block** free_head) { start_index = LARGE_CHUNK_INDEX(free_chunks, section);
Block* free_block = *free_head; for (i = start_index; i < start_index + num_chunks; ++i) {
if (free_block) { assert(section->free_chunk_map[i]);
removeFromLL(free_block); section->free_chunk_map[i] = 0;
return free_block;
} }
return alloc_block(rounded_size, NULL); section->num_free_chunks -= size >> CHUNK_BITS;
assert(section->num_free_chunks >= 0);
return free_chunks;
} }
GCAllocation* SmallArena::_alloc(size_t rounded_size, int bucket_idx) { LargeArena::LargeObj* LargeArena::_alloc(size_t size) {
registerGCManagedBytes(rounded_size); LargeBlock* section;
LargeFreeChunk* free_chunks;
size_t num_chunks;
Block** free_head = &heads[bucket_idx]; size += CHUNK_SIZE - 1;
Block** full_head = &full_heads[bucket_idx]; size &= ~(CHUNK_SIZE - 1);
ThreadBlockCache* cache = thread_caches.get(); num_chunks = size >> CHUNK_BITS;
Block** cache_head = &cache->cache_free_heads[bucket_idx]; assert(size > 0 && size - sizeof(LargeObj) <= ALLOC_SIZE_LIMIT);
assert(num_chunks > 0);
// static __thread int gc_allocs = 0; retry:
// if (++gc_allocs == 128) { if (num_chunks >= NUM_FREE_LISTS) {
// static StatCounter sc_total("gc_allocs"); free_chunks = get_from_size_list(&free_lists[0], size);
// sc_total.log(128); } else {
// gc_allocs = 0; size_t i;
//} for (i = num_chunks; i < NUM_FREE_LISTS; ++i) {
free_chunks = get_from_size_list(&free_lists[i], size);
if (free_chunks)
break;
}
if (!free_chunks)
free_chunks = get_from_size_list(&free_lists[0], size);
}
while (true) { if (free_chunks)
while (Block* cache_block = *cache_head) { return (LargeObj*)free_chunks;
GCAllocation* rtn = allocFromBlock(cache_block);
if (rtn)
return rtn;
removeFromLL(cache_block); section = (LargeBlock*)doMmap(BLOCK_SIZE);
insertIntoLL(&cache->cache_full_heads[bucket_idx], cache_block);
}
// Not very useful to count the cache misses if we don't count the total attempts: if (!section)
// static StatCounter sc_fallback("gc_allocs_cachemiss"); return NULL;
// sc_fallback.log();
LOCK_REGION(heap->lock); free_chunks = (LargeFreeChunk*)((char*)section + CHUNK_SIZE);
free_chunks->size = BLOCK_SIZE - CHUNK_SIZE;
free_chunks->next_size = free_lists[0];
free_lists[0] = free_chunks;
assert(*cache_head == NULL); section->num_free_chunks = LARGE_BLOCK_NUM_CHUNKS;
// should probably be called allocBlock: section->free_chunk_map = (unsigned char*)section + sizeof(LargeBlock);
Block* myblock = claimBlock(rounded_size, &heads[bucket_idx]); assert(sizeof(LargeBlock) + LARGE_BLOCK_NUM_CHUNKS + 1 <= CHUNK_SIZE);
assert(myblock); section->free_chunk_map[0] = 0;
assert(!myblock->next); memset(section->free_chunk_map + 1, 1, LARGE_BLOCK_NUM_CHUNKS);
assert(!myblock->prev);
// printf("%d claimed new block %p with %d objects\n", threading::gettid(), myblock, myblock->numObjects()); section->next = blocks;
blocks = section;
insertIntoLL(cache_head, myblock); goto retry;
}
} }
void SmallArena::_free(GCAllocation* alloc, Block* b) { void LargeArena::_freeLargeObj(LargeObj* obj) {
assert(b == Block::forPointer(alloc)); removeFromLL(obj);
size_t size = b->size; size_t size = obj->size;
int offset = (char*)alloc - (char*)b; LargeBlock* section = LARGE_BLOCK_FOR_OBJ(obj);
assert(offset % size == 0); size_t num_chunks, i, start_index;
int atom_idx = offset / ATOM_SIZE;
assert(!b->isfree.isSet(atom_idx)); size += CHUNK_SIZE - 1;
b->isfree.set(atom_idx); size &= ~(CHUNK_SIZE - 1);
#ifndef NVALGRIND num_chunks = size >> CHUNK_BITS;
// VALGRIND_MEMPOOL_FREE(b, ptr);
#endif
}
void _doFree(GCAllocation* al) { assert(size > 0 && size - sizeof(LargeObj) <= ALLOC_SIZE_LIMIT);
if (VERBOSITY() >= 2) assert(num_chunks > 0);
printf("Freeing %p\n", al->user_data);
#ifndef NVALGRIND section->num_free_chunks += num_chunks;
VALGRIND_DISABLE_ERROR_REPORTING; assert(section->num_free_chunks <= LARGE_BLOCK_NUM_CHUNKS);
#endif
GCKind alloc_kind = al->kind_id;
#ifndef NVALGRIND
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
if (alloc_kind == GCKind::PYTHON) { /*
#ifndef NVALGRIND * We could free the LOS section here if it's empty, but we
VALGRIND_DISABLE_ERROR_REPORTING; * can't unless we also remove its free chunks from the fast
#endif * free lists. Instead, we do it in los_sweep().
Box* b = (Box*)al->user_data; */
#ifndef NVALGRIND
VALGRIND_ENABLE_ERROR_REPORTING;
#endif
ASSERT(b->cls->tp_dealloc == NULL, "%s", getTypeName(b)); start_index = LARGE_CHUNK_INDEX(obj, section);
if (b->cls->simple_destructor) for (i = start_index; i < start_index + num_chunks; ++i) {
b->cls->simple_destructor(b); assert(!section->free_chunk_map[i]);
section->free_chunk_map[i] = 1;
} }
}
void Heap::destroyContents(GCAllocation* al) { add_free_chunk((LargeFreeChunk*)obj, size);
_doFree(al);
} }
void dumpHeapStatistics() { //////
global_heap.dumpHeapStatistics(); /// Huge Arena
}
struct HeapStatistics {
struct TypeStats {
int64_t nallocs;
int64_t nbytes;
TypeStats() : nallocs(0), nbytes(0) {}
void print(const char* name) const { GCAllocation* HugeArena::alloc(size_t size) {
if (nbytes > (1 << 20)) registerGCManagedBytes(size);
printf("%s: %ld allocations for %.1f MB\n", name, nallocs, nbytes * 1.0 / (1 << 20));
else if (nbytes > (1 << 10))
printf("%s: %ld allocations for %.1f KB\n", name, nallocs, nbytes * 1.0 / (1 << 10));
else
printf("%s: %ld allocations for %ld bytes\n", name, nallocs, nbytes);
}
};
std::unordered_map<BoxedClass*, TypeStats> by_cls;
TypeStats conservative, untracked;
TypeStats total;
};
void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
stats->total.nallocs++;
stats->total.nbytes += nbytes;
if (al->kind_id == GCKind::PYTHON) { LOCK_REGION(heap->lock);
Box* b = (Box*)al->user_data;
auto& t = stats->by_cls[b->cls];
t.nallocs++; size_t total_size = size + sizeof(HugeObj);
t.nbytes += nbytes; total_size = (total_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
} else if (al->kind_id == GCKind::CONSERVATIVE) { HugeObj* rtn = (HugeObj*)doMmap(total_size);
stats->conservative.nallocs++; rtn->obj_size = size;
stats->conservative.nbytes += nbytes;
} else if (al->kind_id == GCKind::UNTRACKED) {
stats->untracked.nallocs++;
stats->untracked.nbytes += nbytes;
} else {
RELEASE_ASSERT(0, "%d", (int)al->kind_id);
}
}
// TODO: copy-pasted from freeChain nullNextPrev(rtn);
void SmallArena::getChainStatistics(HeapStatistics* stats, Block** head) { insertIntoLL(&head, rtn);
while (Block* b = *head) {
int num_objects = b->numObjects();
int first_obj = b->minObjIndex();
int atoms_per_obj = b->atomsPerObj();
for (int obj_idx = first_obj; obj_idx < num_objects; obj_idx++) { return rtn->data;
int atom_idx = obj_idx * atoms_per_obj; }
if (b->isfree.isSet(atom_idx)) GCAllocation* HugeArena::realloc(GCAllocation* al, size_t bytes) {
continue; HugeObj* obj = HugeObj::fromAllocation(al);
void* p = &b->atoms[atom_idx]; int capacity = obj->capacity();
GCAllocation* al = reinterpret_cast<GCAllocation*>(p); if (capacity >= bytes && capacity < bytes * 2)
return al;
addStatistic(stats, al, b->size); GCAllocation* rtn = heap->alloc(bytes);
} memcpy(rtn, al, std::min(bytes, obj->obj_size));
head = &b->next; _freeHugeObj(obj);
} return rtn;
} }
// TODO: copy-pasted from freeUnmarked() void HugeArena::free(GCAllocation* al) {
void SmallArena::getStatistics(HeapStatistics* stats) { _freeHugeObj(HugeObj::fromAllocation(al));
thread_caches.forEachValue([this, stats](ThreadBlockCache* cache) {
for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
Block* h = cache->cache_free_heads[bidx];
getChainStatistics(stats, &cache->cache_free_heads[bidx]);
getChainStatistics(stats, &cache->cache_full_heads[bidx]);
}
});
for (int bidx = 0; bidx < NUM_BUCKETS; bidx++) {
getChainStatistics(stats, &heads[bidx]);
getChainStatistics(stats, &full_heads[bidx]);
}
} }
void LargeArena::getStatistics(HeapStatistics* stats) { GCAllocation* HugeArena::allocationFrom(void* ptr) {
LargeObj* cur = head; HugeObj* cur = head;
while (cur) { while (cur) {
GCAllocation* al = cur->data; if (ptr >= cur && ptr < &cur->data[cur->obj_size])
addStatistic(stats, al, cur->size); return &cur->data[0];
cur = cur->next; cur = cur->next;
} }
return NULL;
}
void HugeArena::freeUnmarked() {
sweepList(head, [this](HugeObj* ptr) { _freeHugeObj(ptr); });
} }
void HugeArena::getStatistics(HeapStatistics* stats) { void HugeArena::getStatistics(HeapStatistics* stats) {
...@@ -757,23 +783,12 @@ void HugeArena::getStatistics(HeapStatistics* stats) { ...@@ -757,23 +783,12 @@ void HugeArena::getStatistics(HeapStatistics* stats) {
} }
} }
void Heap::dumpHeapStatistics() { void HugeArena::_freeHugeObj(HugeObj* lobj) {
threading::GLPromoteRegion _lock; removeFromLL(lobj);
int r = munmap(lobj, lobj->mmap_size());
HeapStatistics stats; assert(r == 0);
small_arena.getStatistics(&stats);
large_arena.getStatistics(&stats);
huge_arena.getStatistics(&stats);
stats.conservative.print("conservative");
stats.untracked.print("untracked");
for (const auto& p : stats.by_cls) {
p.second.print(getFullNameOfClass(p.first).c_str());
}
stats.total.print("Total");
printf("\n");
} }
} // namespace gc } // namespace gc
} // namespace pyston } // namespace pyston
...@@ -65,17 +65,20 @@ inline void clearMark(GCAllocation* header) { ...@@ -65,17 +65,20 @@ inline void clearMark(GCAllocation* header) {
#define PAGE_SIZE 4096 #define PAGE_SIZE 4096
template <uintptr_t start> class Arena { template <uintptr_t arena_start, uintptr_t arena_size> class Arena {
private: private:
void* cur; void* cur;
void* end;
protected: protected:
Arena() : cur((void*)start) {} Arena() : cur((void*)arena_start), end((void*)(arena_start + arena_size)) {}
public: public:
void* doMmap(size_t size) { void* doMmap(size_t size) {
assert(size % PAGE_SIZE == 0); assert(size % PAGE_SIZE == 0);
assert(((uint8_t*)cur + size) < end && "arena full");
void* mrtn = mmap(cur, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); void* mrtn = mmap(cur, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
assert((uintptr_t)mrtn != -1 && "failed to allocate memory from OS"); assert((uintptr_t)mrtn != -1 && "failed to allocate memory from OS");
ASSERT(mrtn == cur, "%p %p\n", mrtn, cur); ASSERT(mrtn == cur, "%p %p\n", mrtn, cur);
...@@ -83,9 +86,10 @@ public: ...@@ -83,9 +86,10 @@ public:
return mrtn; return mrtn;
} }
bool contains(void* addr) { return (void*)start <= addr && addr < cur; } bool contains(void* addr) { return (void*)arena_start <= addr && addr < cur; }
}; };
constexpr uintptr_t ARENA_SIZE = 0x1000000000L;
constexpr uintptr_t SMALL_ARENA_START = 0x1270000000L; constexpr uintptr_t SMALL_ARENA_START = 0x1270000000L;
constexpr uintptr_t LARGE_ARENA_START = 0x2270000000L; constexpr uintptr_t LARGE_ARENA_START = 0x2270000000L;
constexpr uintptr_t HUGE_ARENA_START = 0x3270000000L; constexpr uintptr_t HUGE_ARENA_START = 0x3270000000L;
...@@ -94,8 +98,8 @@ constexpr uintptr_t HUGE_ARENA_START = 0x3270000000L; ...@@ -94,8 +98,8 @@ constexpr uintptr_t HUGE_ARENA_START = 0x3270000000L;
// //
// The SmallArena allocates objects <= 3584 bytes. // The SmallArena allocates objects <= 3584 bytes.
// //
// it uses segregated-fit allocation, and each block contains free // it uses segregated-fit allocation, and each block contains a free
// bitmap for objects of a given size (assigned to the block) // bitmap for objects of a given size (constant for the block)
// //
static const size_t sizes[] = { static const size_t sizes[] = {
16, 32, 48, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 16, 32, 48, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384,
...@@ -103,8 +107,20 @@ static const size_t sizes[] = { ...@@ -103,8 +107,20 @@ static const size_t sizes[] = {
}; };
static constexpr size_t NUM_BUCKETS = sizeof(sizes) / sizeof(sizes[0]); static constexpr size_t NUM_BUCKETS = sizeof(sizes) / sizeof(sizes[0]);
class SmallArena : public Arena<SMALL_ARENA_START> {
class SmallArena : public Arena<SMALL_ARENA_START, ARENA_SIZE> {
public: public:
SmallArena(Heap* heap) : Arena(), heap(heap), thread_caches(heap, this) {}
GCAllocation* __attribute__((__malloc__)) alloc(size_t bytes);
GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
void free(GCAllocation* al);
GCAllocation* allocationFrom(void* ptr);
void freeUnmarked();
void getStatistics(HeapStatistics* stats);
private: private:
template <int N> class Bitmap { template <int N> class Bitmap {
static_assert(N % 64 == 0, ""); static_assert(N % 64 == 0, "");
...@@ -205,8 +221,7 @@ private: ...@@ -205,8 +221,7 @@ private:
static_assert(offsetof(Block, _header_end) >= BLOCK_HEADER_SIZE, "bad header size"); static_assert(offsetof(Block, _header_end) >= BLOCK_HEADER_SIZE, "bad header size");
static_assert(offsetof(Block, _header_end) <= BLOCK_HEADER_SIZE, "bad header size"); static_assert(offsetof(Block, _header_end) <= BLOCK_HEADER_SIZE, "bad header size");
// forward (public) definition of ThreadBlockCache so we can reference it both in this class (privately) and in Heap
// (for a friend ref).
struct ThreadBlockCache { struct ThreadBlockCache {
Heap* heap; Heap* heap;
SmallArena* small; SmallArena* small;
...@@ -221,7 +236,6 @@ private: ...@@ -221,7 +236,6 @@ private:
}; };
Block* heads[NUM_BUCKETS]; Block* heads[NUM_BUCKETS];
Block* full_heads[NUM_BUCKETS]; Block* full_heads[NUM_BUCKETS];
...@@ -231,71 +245,46 @@ private: ...@@ -231,71 +245,46 @@ private:
// TODO only use thread caches if we're in GRWL mode? // TODO only use thread caches if we're in GRWL mode?
threading::PerThreadSet<ThreadBlockCache, Heap*, SmallArena*> thread_caches; threading::PerThreadSet<ThreadBlockCache, Heap*, SmallArena*> thread_caches;
Block* _allocBlock(uint64_t size, Block** prev);
Block* alloc_block(uint64_t size, Block** prev); GCAllocation* _allocFromBlock(Block* b);
GCAllocation* allocFromBlock(Block* b); Block* _claimBlock(size_t rounded_size, Block** free_head);
Block* claimBlock(size_t rounded_size, Block** free_head); Block** _freeChain(Block** head);
void insertIntoLL(Block** next_pointer, Block* next); void _getChainStatistics(HeapStatistics* stats, Block** head);
void removeFromLL(Block* b);
Block** freeChain(Block** head);
void getChainStatistics(HeapStatistics* stats, Block** head);
GCAllocation* __attribute__((__malloc__)) _alloc(size_t bytes, int bucket_idx); GCAllocation* __attribute__((__malloc__)) _alloc(size_t bytes, int bucket_idx);
void _free(GCAllocation* al, Block* b);
public:
SmallArena(Heap* heap) : Arena(), heap(heap), thread_caches(heap, this) {}
GCAllocation* __attribute__((__malloc__)) alloc(size_t bytes) {
if (bytes <= 16)
return _alloc(16, 0);
else if (bytes <= 32)
return _alloc(32, 1);
else {
for (int i = 2; i < NUM_BUCKETS; i++) {
if (sizes[i] >= bytes) {
return _alloc(sizes[i], i);
}
}
return NULL;
}
}
GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
void free(GCAllocation* al) {
Block* b = Block::forPointer(al);
_free(al, b);
}
void getStatistics(HeapStatistics* stats);
GCAllocation* allocationFrom(void* ptr);
void freeUnmarked();
}; };
// //
// The LargeArena allocates objects where 3584 < size <1024*1024 bytes. // The LargeArena allocates objects where 3584 < size <1024*1024-CHUNK_SIZE-sizeof(LargeObject) bytes.
// //
// it maintains a set of size-segregated free lists, and a special // it maintains a set of size-segregated free lists, and a special
// free list for larger objects. If the free list specific to a given // free list for larger objects. If the free list specific to a given
// size has no entries, we search the large free list. // size has no entries, we search the large free list.
// //
class LargeArena : public Arena<LARGE_ARENA_START> { // Blocks of 1meg are mmap'ed individually, and carved up as needed.
struct LargeFreeChunk { //
LargeFreeChunk* next_size; class LargeArena : public Arena<LARGE_ARENA_START, ARENA_SIZE> {
size_t size; private:
};
struct LargeBlock { struct LargeBlock {
LargeBlock* next; LargeBlock* next;
size_t num_free_chunks; size_t num_free_chunks;
unsigned char* free_chunk_map; unsigned char* free_chunk_map;
}; };
struct LargeFreeChunk {
LargeFreeChunk* next_size;
size_t size;
};
struct LargeObj { struct LargeObj {
LargeObj* next, **prev; LargeObj* next, **prev;
size_t size; size_t size;
GCAllocation data[0]; GCAllocation data[0];
static LargeObj* fromAllocation(GCAllocation* alloc) {
char* rtn = (char*)alloc - offsetof(LargeObj, data);
return reinterpret_cast<LargeObj*>(rtn);
}
}; };
/* /*
...@@ -311,20 +300,18 @@ class LargeArena : public Arena<LARGE_ARENA_START> { ...@@ -311,20 +300,18 @@ class LargeArena : public Arena<LARGE_ARENA_START> {
static constexpr int NUM_FREE_LISTS = 32; static constexpr int NUM_FREE_LISTS = 32;
void add_free_chunk(LargeFreeChunk* free_chunks, size_t size); Heap* heap;
LargeFreeChunk* get_from_size_list(LargeFreeChunk** list, size_t size);
LargeObj* _allocInternal(size_t size);
void _freeInternal(LargeObj* obj, size_t size);
void _free(LargeObj* obj);
LargeObj* head; LargeObj* head;
LargeBlock* blocks; LargeBlock* blocks;
LargeFreeChunk* free_lists[NUM_FREE_LISTS]; /* 0 is for larger sizes */ LargeFreeChunk* free_lists[NUM_FREE_LISTS]; /* 0 is for larger sizes */
Heap* heap; void add_free_chunk(LargeFreeChunk* free_chunks, size_t size);
LargeFreeChunk* get_from_size_list(LargeFreeChunk** list, size_t size);
LargeObj* _alloc(size_t size);
void _freeLargeObj(LargeObj* obj);
public: public:
LargeArena(Heap* heap) : head(NULL), blocks(NULL), heap(heap) {} LargeArena(Heap* heap) : heap(heap), head(NULL), blocks(NULL) {}
/* Largest object that can be allocated in a large block. */ /* Largest object that can be allocated in a large block. */
static constexpr size_t ALLOC_SIZE_LIMIT = BLOCK_SIZE - CHUNK_SIZE - sizeof(LargeObj); static constexpr size_t ALLOC_SIZE_LIMIT = BLOCK_SIZE - CHUNK_SIZE - sizeof(LargeObj);
...@@ -333,9 +320,9 @@ public: ...@@ -333,9 +320,9 @@ public:
GCAllocation* realloc(GCAllocation* alloc, size_t bytes); GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
void free(GCAllocation* alloc); void free(GCAllocation* alloc);
GCAllocation* allocationFrom(void* ptr);
void freeUnmarked(); void freeUnmarked();
GCAllocation* allocationFrom(void* ptr);
void getStatistics(HeapStatistics* stats); void getStatistics(HeapStatistics* stats);
}; };
...@@ -343,7 +330,20 @@ public: ...@@ -343,7 +330,20 @@ public:
// //
// Objects are allocated with individual mmap() calls, and kept in a // Objects are allocated with individual mmap() calls, and kept in a
// linked list. They are not reused. // linked list. They are not reused.
class HugeArena : public Arena<HUGE_ARENA_START> { class HugeArena : public Arena<HUGE_ARENA_START, ARENA_SIZE> {
public:
HugeArena(Heap* heap) : heap(heap) {}
GCAllocation* __attribute__((__malloc__)) alloc(size_t bytes);
GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
void free(GCAllocation* alloc);
GCAllocation* allocationFrom(void* ptr);
void freeUnmarked();
void getStatistics(HeapStatistics* stats);
private:
struct HugeObj { struct HugeObj {
HugeObj* next, **prev; HugeObj* next, **prev;
size_t obj_size; size_t obj_size;
...@@ -369,18 +369,6 @@ class HugeArena : public Arena<HUGE_ARENA_START> { ...@@ -369,18 +369,6 @@ class HugeArena : public Arena<HUGE_ARENA_START> {
HugeObj* head; HugeObj* head;
Heap* heap; Heap* heap;
public:
HugeArena(Heap* heap) : heap(heap) {}
GCAllocation* __attribute__((__malloc__)) alloc(size_t bytes);
GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
void free(GCAllocation* alloc);
void freeUnmarked();
GCAllocation* allocationFrom(void* ptr);
void getStatistics(HeapStatistics* stats);
}; };
...@@ -420,10 +408,10 @@ public: ...@@ -420,10 +408,10 @@ public:
return small_arena.alloc(bytes); return small_arena.alloc(bytes);
} }
void destroyContents(GCAllocation* alloc); void destructContents(GCAllocation* alloc);
void free(GCAllocation* alloc) { void free(GCAllocation* alloc) {
destroyContents(alloc); destructContents(alloc);
if (large_arena.contains(alloc)) { if (large_arena.contains(alloc)) {
large_arena.free(alloc); large_arena.free(alloc);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment