Commit 14d1b9a6 authored by Matthew Auld's avatar Matthew Auld Committed by Chris Wilson

drm/i915: buddy allocator

Simple buddy allocator. We want to allocate properly aligned
power-of-two blocks to promote usage of huge-pages for the GTT, so 64K,
2M and possibly even 1G. While we do support allocating stuff at a
specific offset, it is more intended for preallocating portions of the
address space, say for an initial framebuffer, for other uses drm_mm is
probably a much better fit. Anyway, hopefully this can all be thrown
away if we eventually move to having the core MM manage device memory.
Signed-off-by: default avatarMatthew Auld <matthew.auld@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190809202926.14545-2-matthew.auld@intel.com
parent 05f219d7
...@@ -127,6 +127,7 @@ gem-y += \ ...@@ -127,6 +127,7 @@ gem-y += \
i915-y += \ i915-y += \
$(gem-y) \ $(gem-y) \
i915_active.o \ i915_active.o \
i915_buddy.o \
i915_cmd_parser.o \ i915_cmd_parser.o \
i915_gem_evict.o \ i915_gem_evict.o \
i915_gem_fence_reg.o \ i915_gem_fence_reg.o \
......
// SPDX-License-Identifier: MIT
/*
* Copyright © 2019 Intel Corporation
*/
#include <linux/slab.h>
#include "i915_buddy.h"
#include "i915_gem.h"
#include "i915_globals.h"
#include "i915_utils.h"
static struct i915_global_block {
struct i915_global base;
struct kmem_cache *slab_blocks;
} global;
static void i915_global_buddy_shrink(void)
{
kmem_cache_shrink(global.slab_blocks);
}
static void i915_global_buddy_exit(void)
{
kmem_cache_destroy(global.slab_blocks);
}
static struct i915_global_block global = { {
.shrink = i915_global_buddy_shrink,
.exit = i915_global_buddy_exit,
} };
int __init i915_global_buddy_init(void)
{
global.slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
if (!global.slab_blocks)
return -ENOMEM;
return 0;
}
static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_block *parent,
unsigned int order,
u64 offset)
{
struct i915_buddy_block *block;
block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
if (!block)
return NULL;
block->header = offset;
block->header |= order;
block->parent = parent;
return block;
}
static void i915_block_free(struct i915_buddy_block *block)
{
kmem_cache_free(global.slab_blocks, block);
}
static void mark_allocated(struct i915_buddy_block *block)
{
block->header &= ~I915_BUDDY_HEADER_STATE;
block->header |= I915_BUDDY_ALLOCATED;
list_del(&block->link);
}
static void mark_free(struct i915_buddy_mm *mm,
struct i915_buddy_block *block)
{
block->header &= ~I915_BUDDY_HEADER_STATE;
block->header |= I915_BUDDY_FREE;
list_add(&block->link,
&mm->free_list[i915_buddy_block_order(block)]);
}
static void mark_split(struct i915_buddy_block *block)
{
block->header &= ~I915_BUDDY_HEADER_STATE;
block->header |= I915_BUDDY_SPLIT;
list_del(&block->link);
}
int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
{
unsigned int i;
u64 offset;
if (size < chunk_size)
return -EINVAL;
if (chunk_size < PAGE_SIZE)
return -EINVAL;
if (!is_power_of_2(chunk_size))
return -EINVAL;
size = round_down(size, chunk_size);
mm->size = size;
mm->chunk_size = chunk_size;
mm->max_order = ilog2(size) - ilog2(chunk_size);
GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
mm->free_list = kmalloc_array(mm->max_order + 1,
sizeof(struct list_head),
GFP_KERNEL);
if (!mm->free_list)
return -ENOMEM;
for (i = 0; i <= mm->max_order; ++i)
INIT_LIST_HEAD(&mm->free_list[i]);
mm->n_roots = hweight64(size);
mm->roots = kmalloc_array(mm->n_roots,
sizeof(struct i915_buddy_block *),
GFP_KERNEL);
if (!mm->roots)
goto out_free_list;
offset = 0;
i = 0;
/*
* Split into power-of-two blocks, in case we are given a size that is
* not itself a power-of-two.
*/
do {
struct i915_buddy_block *root;
unsigned int order;
u64 root_size;
root_size = rounddown_pow_of_two(size);
order = ilog2(root_size) - ilog2(chunk_size);
root = i915_block_alloc(NULL, order, offset);
if (!root)
goto out_free_roots;
mark_free(mm, root);
GEM_BUG_ON(i > mm->max_order);
GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size);
mm->roots[i] = root;
offset += root_size;
size -= root_size;
i++;
} while (size);
return 0;
out_free_roots:
while (i--)
i915_block_free(mm->roots[i]);
kfree(mm->roots);
out_free_list:
kfree(mm->free_list);
return -ENOMEM;
}
void i915_buddy_fini(struct i915_buddy_mm *mm)
{
int err = 0;
int i;
for (i = 0; i < mm->n_roots; ++i) {
if (!i915_buddy_block_is_free(mm->roots[i])) {
err = -EBUSY;
continue;
}
i915_block_free(mm->roots[i]);
}
kfree(mm->roots);
kfree(mm->free_list);
}
static int split_block(struct i915_buddy_mm *mm,
struct i915_buddy_block *block)
{
unsigned int block_order = i915_buddy_block_order(block) - 1;
u64 offset = i915_buddy_block_offset(block);
GEM_BUG_ON(!i915_buddy_block_is_free(block));
GEM_BUG_ON(!i915_buddy_block_order(block));
block->left = i915_block_alloc(block, block_order, offset);
if (!block->left)
return -ENOMEM;
block->right = i915_block_alloc(block, block_order,
offset + (mm->chunk_size << block_order));
if (!block->right) {
i915_block_free(block->left);
return -ENOMEM;
}
mark_free(mm, block->left);
mark_free(mm, block->right);
mark_split(block);
return 0;
}
static struct i915_buddy_block *
get_buddy(struct i915_buddy_block *block)
{
struct i915_buddy_block *parent;
parent = block->parent;
if (!parent)
return NULL;
if (parent->left == block)
return parent->right;
return parent->left;
}
static void __i915_buddy_free(struct i915_buddy_mm *mm,
struct i915_buddy_block *block)
{
struct i915_buddy_block *parent;
while ((parent = block->parent)) {
struct i915_buddy_block *buddy;
buddy = get_buddy(block);
if (!i915_buddy_block_is_free(buddy))
break;
list_del(&buddy->link);
i915_block_free(block);
i915_block_free(buddy);
block = parent;
}
mark_free(mm, block);
}
void i915_buddy_free(struct i915_buddy_mm *mm,
struct i915_buddy_block *block)
{
GEM_BUG_ON(!i915_buddy_block_is_allocated(block));
__i915_buddy_free(mm, block);
}
void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects)
{
struct i915_buddy_block *block, *on;
list_for_each_entry_safe(block, on, objects, link)
i915_buddy_free(mm, block);
INIT_LIST_HEAD(objects);
}
/*
* Allocate power-of-two block. The order value here translates to:
*
* 0 = 2^0 * mm->chunk_size
* 1 = 2^1 * mm->chunk_size
* 2 = 2^2 * mm->chunk_size
* ...
*/
struct i915_buddy_block *
i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order)
{
struct i915_buddy_block *block = NULL;
unsigned int i;
int err;
for (i = order; i <= mm->max_order; ++i) {
block = list_first_entry_or_null(&mm->free_list[i],
struct i915_buddy_block,
link);
if (block)
break;
}
if (!block)
return ERR_PTR(-ENOSPC);
GEM_BUG_ON(!i915_buddy_block_is_free(block));
while (i != order) {
err = split_block(mm, block);
if (unlikely(err))
goto out_free;
/* Go low */
block = block->left;
i--;
}
mark_allocated(block);
return block;
out_free:
__i915_buddy_free(mm, block);
return ERR_PTR(err);
}
static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
{
return s1 <= e2 && e1 >= s2;
}
static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
{
return s1 <= s2 && e1 >= e2;
}
/*
* Allocate range. Note that it's safe to chain together multiple alloc_ranges
* with the same blocks list.
*
* Intended for pre-allocating portions of the address space, for example to
* reserve a block for the initial framebuffer or similar, hence the expectation
* here is that i915_buddy_alloc() is still the main vehicle for
* allocations, so if that's not the case then the drm_mm range allocator is
* probably a much better fit, and so you should probably go use that instead.
*/
int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
struct list_head *blocks,
u64 start, u64 size)
{
struct i915_buddy_block *block;
struct i915_buddy_block *buddy;
LIST_HEAD(allocated);
LIST_HEAD(dfs);
u64 end;
int err;
int i;
if (size < mm->chunk_size)
return -EINVAL;
if (!IS_ALIGNED(size | start, mm->chunk_size))
return -EINVAL;
if (range_overflows(start, size, mm->size))
return -EINVAL;
for (i = 0; i < mm->n_roots; ++i)
list_add_tail(&mm->roots[i]->tmp_link, &dfs);
end = start + size - 1;
do {
u64 block_start;
u64 block_end;
block = list_first_entry_or_null(&dfs,
struct i915_buddy_block,
tmp_link);
if (!block)
break;
list_del(&block->tmp_link);
block_start = i915_buddy_block_offset(block);
block_end = block_start + i915_buddy_block_size(mm, block) - 1;
if (!overlaps(start, end, block_start, block_end))
continue;
if (i915_buddy_block_is_allocated(block)) {
err = -ENOSPC;
goto err_free;
}
if (contains(start, end, block_start, block_end)) {
if (!i915_buddy_block_is_free(block)) {
err = -ENOSPC;
goto err_free;
}
mark_allocated(block);
list_add_tail(&block->link, &allocated);
continue;
}
if (!i915_buddy_block_is_split(block)) {
err = split_block(mm, block);
if (unlikely(err))
goto err_undo;
}
list_add(&block->right->tmp_link, &dfs);
list_add(&block->left->tmp_link, &dfs);
} while (1);
list_splice_tail(&allocated, blocks);
return 0;
err_undo:
/*
* We really don't want to leave around a bunch of split blocks, since
* bigger is better, so make sure we merge everything back before we
* free the allocated blocks.
*/
buddy = get_buddy(block);
if (buddy &&
(i915_buddy_block_is_free(block) &&
i915_buddy_block_is_free(buddy)))
__i915_buddy_free(mm, block);
err_free:
i915_buddy_free_list(mm, &allocated);
return err;
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/i915_buddy.c"
#endif
/* SPDX-License-Identifier: MIT */
/*
* Copyright © 2019 Intel Corporation
*/
#ifndef __I915_BUDDY_H__
#define __I915_BUDDY_H__
#include <linux/bitops.h>
#include <linux/list.h>
struct i915_buddy_block {
#define I915_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12)
#define I915_BUDDY_HEADER_STATE GENMASK_ULL(11, 10)
#define I915_BUDDY_ALLOCATED (1 << 10)
#define I915_BUDDY_FREE (2 << 10)
#define I915_BUDDY_SPLIT (3 << 10)
#define I915_BUDDY_HEADER_ORDER GENMASK_ULL(9, 0)
u64 header;
struct i915_buddy_block *left;
struct i915_buddy_block *right;
struct i915_buddy_block *parent;
void *private; /* owned by creator */
/*
* While the block is allocated by the user through i915_buddy_alloc*,
* the user has ownership of the link, for example to maintain within
* a list, if so desired. As soon as the block is freed with
* i915_buddy_free* ownership is given back to the mm.
*/
struct list_head link;
struct list_head tmp_link;
};
#define I915_BUDDY_MAX_ORDER I915_BUDDY_HEADER_ORDER
/*
* Binary Buddy System.
*
* Locking should be handled by the user, a simple mutex around
* i915_buddy_alloc* and i915_buddy_free* should suffice.
*/
struct i915_buddy_mm {
/* Maintain a free list for each order. */
struct list_head *free_list;
/*
* Maintain explicit binary tree(s) to track the allocation of the
* address space. This gives us a simple way of finding a buddy block
* and performing the potentially recursive merge step when freeing a
* block. Nodes are either allocated or free, in which case they will
* also exist on the respective free list.
*/
struct i915_buddy_block **roots;
/*
* Anything from here is public, and remains static for the lifetime of
* the mm. Everything above is considered do-not-touch.
*/
unsigned int n_roots;
unsigned int max_order;
/* Must be at least PAGE_SIZE */
u64 chunk_size;
u64 size;
};
static inline u64
i915_buddy_block_offset(struct i915_buddy_block *block)
{
return block->header & I915_BUDDY_HEADER_OFFSET;
}
static inline unsigned int
i915_buddy_block_order(struct i915_buddy_block *block)
{
return block->header & I915_BUDDY_HEADER_ORDER;
}
static inline unsigned int
i915_buddy_block_state(struct i915_buddy_block *block)
{
return block->header & I915_BUDDY_HEADER_STATE;
}
static inline bool
i915_buddy_block_is_allocated(struct i915_buddy_block *block)
{
return i915_buddy_block_state(block) == I915_BUDDY_ALLOCATED;
}
static inline bool
i915_buddy_block_is_free(struct i915_buddy_block *block)
{
return i915_buddy_block_state(block) == I915_BUDDY_FREE;
}
static inline bool
i915_buddy_block_is_split(struct i915_buddy_block *block)
{
return i915_buddy_block_state(block) == I915_BUDDY_SPLIT;
}
static inline u64
i915_buddy_block_size(struct i915_buddy_mm *mm,
struct i915_buddy_block *block)
{
return mm->chunk_size << i915_buddy_block_order(block);
}
int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size);
void i915_buddy_fini(struct i915_buddy_mm *mm);
struct i915_buddy_block *
i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order);
int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
struct list_head *blocks,
u64 start, u64 size);
void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block);
void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects);
#endif
...@@ -62,6 +62,7 @@ static void __i915_globals_cleanup(void) ...@@ -62,6 +62,7 @@ static void __i915_globals_cleanup(void)
static __initconst int (* const initfn[])(void) = { static __initconst int (* const initfn[])(void) = {
i915_global_active_init, i915_global_active_init,
i915_global_buddy_init,
i915_global_context_init, i915_global_context_init,
i915_global_gem_context_init, i915_global_gem_context_init,
i915_global_objects_init, i915_global_objects_init,
......
...@@ -27,6 +27,7 @@ void i915_globals_exit(void); ...@@ -27,6 +27,7 @@ void i915_globals_exit(void);
/* constructors */ /* constructors */
int i915_global_active_init(void); int i915_global_active_init(void);
int i915_global_buddy_init(void);
int i915_global_context_init(void); int i915_global_context_init(void);
int i915_global_gem_context_init(void); int i915_global_gem_context_init(void);
int i915_global_objects_init(void); int i915_global_objects_init(void);
......
This diff is collapsed.
...@@ -25,3 +25,4 @@ selftest(evict, i915_gem_evict_mock_selftests) ...@@ -25,3 +25,4 @@ selftest(evict, i915_gem_evict_mock_selftests)
selftest(gtt, i915_gem_gtt_mock_selftests) selftest(gtt, i915_gem_gtt_mock_selftests)
selftest(hugepages, i915_gem_huge_page_mock_selftests) selftest(hugepages, i915_gem_huge_page_mock_selftests)
selftest(contexts, i915_gem_context_mock_selftests) selftest(contexts, i915_gem_context_mock_selftests)
selftest(buddy, i915_buddy_mock_selftests)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment