drm/ttm: add transparent huge page support for DMA allocations v2

Try to allocate huge pages when it makes sense.

v2: fix comment and use ifdef
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/ttm/ttm_page_alloc_dma.c

@@ -60,21 +60,25 @@
 #define NUM_PAGES_TO_ALLOC		(PAGE_SIZE/sizeof(struct page *))
 #define SMALL_ALLOCATION		4
 #define FREE_ALL_PAGES			(~0U)
+#define VADDR_FLAG_HUGE_POOL		1UL
 
 enum pool_type {
 	IS_UNDEFINED	= 0,
 	IS_WC		= 1 << 1,
 	IS_UC		= 1 << 2,
 	IS_CACHED	= 1 << 3,
-	IS_DMA32	= 1 << 4
+	IS_DMA32	= 1 << 4,
+	IS_HUGE		= 1 << 5
 };
 
 /*
- * The pool structure. There are usually six pools:
+ * The pool structure. There are up to nine pools:
  *  - generic (not restricted to DMA32):
  *      - write combined, uncached, cached.
  *  - dma32 (up to 2^32 - so up 4GB):
  *      - write combined, uncached, cached.
+ *  - huge (not restricted to DMA32):
+ *      - write combined, uncached, cached.
  * for each 'struct device'. The 'cached' is for pages that are actively used.
  * The other ones can be shrunk by the shrinker API if neccessary.
  * @pools: The 'struct device->dma_pools' link.
@@ -114,13 +118,14 @@ struct dma_pool {
  * The accounting page keeping track of the allocated page along with
  * the DMA address.
  * @page_list: The link to the 'page_list' in 'struct dma_pool'.
- * @vaddr: The virtual address of the page
+ * @vaddr: The virtual address of the page and a flag if the page belongs to a
+ * huge pool
  * @dma: The bus address of the page. If the page is not allocated
  *   via the DMA API, it will be -1.
  */
 struct dma_page {
 	struct list_head page_list;
-	void *vaddr;
+	unsigned long vaddr;
 	struct page *p;
 	dma_addr_t dma;
 };
@@ -319,7 +324,8 @@ static int ttm_set_pages_caching(struct dma_pool *pool,
 static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)
 {
 	dma_addr_t dma = d_page->dma;
-	dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma);
+	d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL;
+	dma_free_coherent(pool->dev, pool->size, (void *)d_page->vaddr, dma);
 
 	kfree(d_page);
 	d_page = NULL;
@@ -327,19 +333,22 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)
 static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool)
 {
 	struct dma_page *d_page;
+	void *vaddr;
 
 	d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL);
 	if (!d_page)
 		return NULL;
 
-	d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size,
-					   &d_page->dma,
-					   pool->gfp_flags);
-	if (d_page->vaddr) {
-		if (is_vmalloc_addr(d_page->vaddr))
-			d_page->p = vmalloc_to_page(d_page->vaddr);
+	vaddr = dma_alloc_coherent(pool->dev, pool->size, &d_page->dma,
+				   pool->gfp_flags);
+	if (vaddr) {
+		if (is_vmalloc_addr(vaddr))
+			d_page->p = vmalloc_to_page(vaddr);
 		else
-			d_page->p = virt_to_page(d_page->vaddr);
+			d_page->p = virt_to_page(vaddr);
+		d_page->vaddr = (unsigned long)vaddr;
+		if (pool->type & IS_HUGE)
+			d_page->vaddr |= VADDR_FLAG_HUGE_POOL;
 	} else {
 		kfree(d_page);
 		d_page = NULL;
@@ -371,11 +380,40 @@ static void ttm_pool_update_free_locked(struct dma_pool *pool,
 }
 
 /* set memory back to wb and free the pages. */
+static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)
+{
+	struct page *page = d_page->p;
+	unsigned i, num_pages;
+	int ret;
+
+	/* Don't set WB on WB page pool. */
+	if (!(pool->type & IS_CACHED)) {
+		num_pages = pool->size / PAGE_SIZE;
+		for (i = 0; i < num_pages; ++i, ++page) {
+			ret = set_pages_array_wb(&page, 1);
+			if (ret) {
+				pr_err("%s: Failed to set %d pages to wb!\n",
+				       pool->dev_name, 1);
+			}
+		}
+	}
+
+	list_del(&d_page->page_list);
+	__ttm_dma_free_page(pool, d_page);
+}
+
 static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,
 			      struct page *pages[], unsigned npages)
 {
 	struct dma_page *d_page, *tmp;
 
+	if (pool->type & IS_HUGE) {
+		list_for_each_entry_safe(d_page, tmp, d_pages, page_list)
+			ttm_dma_page_put(pool, d_page);
+
+		return;
+	}
+
 	/* Don't set WB on WB page pool. */
 	if (npages && !(pool->type & IS_CACHED) &&
 	    set_pages_array_wb(pages, npages))
@@ -388,17 +426,6 @@ static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,
 	}
 }
 
-static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)
-{
-	/* Don't set WB on WB page pool. */
-	if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1))
-		pr_err("%s: Failed to set %d pages to wb!\n",
-		       pool->dev_name, 1);
-
-	list_del(&d_page->page_list);
-	__ttm_dma_free_page(pool, d_page);
-}
-
 /*
  * Free pages from pool.
  *
@@ -567,8 +594,8 @@ static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data)
 static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags,
 					  enum pool_type type)
 {
-	char *n[] = {"wc", "uc", "cached", " dma32", "unknown",};
-	enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED};
+	const char *n[] = {"wc", "uc", "cached", " dma32", "huge"};
+	enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE};
 	struct device_pools *sec_pool = NULL;
 	struct dma_pool *pool = NULL, **ptr;
 	unsigned i;
@@ -605,11 +632,18 @@ static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags,
 	pool->npages_free = pool->npages_in_use = 0;
 	pool->nfrees = 0;
 	pool->gfp_flags = flags;
-	pool->size = PAGE_SIZE;
+	if (type & IS_HUGE)
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+		pool->size = HPAGE_PMD_SIZE;
+#else
+		BUG();
+#endif
+	else
+		pool->size = PAGE_SIZE;
 	pool->type = type;
 	pool->nrefills = 0;
 	p = pool->name;
-	for (i = 0; i < 5; i++) {
+	for (i = 0; i < ARRAY_SIZE(t); i++) {
 		if (type & t[i]) {
 			p += snprintf(p, sizeof(pool->name) - (p - pool->name),
 				      "%s", n[i]);
@@ -713,7 +747,7 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
 	struct dma_page *dma_p;
 	struct page *p;
 	int r = 0;
-	unsigned i, cpages;
+	unsigned i, j, npages, cpages;
 	unsigned max_cpages = min(count,
 			(unsigned)(PAGE_SIZE/sizeof(struct page *)));
 
@@ -751,28 +785,32 @@ static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,
 			goto out;
 		}
 		p = dma_p->p;
+		list_add(&dma_p->page_list, d_pages);
+
 #ifdef CONFIG_HIGHMEM
 		/* gfp flags of highmem page should never be dma32 so we
 		 * we should be fine in such case
 		 */
-		if (!PageHighMem(p))
+		if (PageHighMem(p))
+			continue;
 #endif
-		{
-			caching_array[cpages++] = p;
+
+		npages = pool->size / PAGE_SIZE;
+		for (j = 0; j < npages; ++j) {
+			caching_array[cpages++] = p + j;
 			if (cpages == max_cpages) {
 				/* Note: Cannot hold the spinlock */
 				r = ttm_set_pages_caching(pool, caching_array,
-						 cpages);
+							  cpages);
 				if (r) {
 					ttm_dma_handle_caching_state_failure(
-						pool, d_pages, caching_array,
-						cpages);
+					     pool, d_pages, caching_array,
+					     cpages);
 					goto out;
 				}
 				cpages = 0;
 			}
 		}
-		list_add(&dma_p->page_list, d_pages);
 	}
 
 	if (cpages) {
@@ -860,6 +898,26 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool,
 	return r;
 }
 
+static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge)
+{
+	struct ttm_tt *ttm = &ttm_dma->ttm;
+	gfp_t gfp_flags;
+
+	if (ttm->page_flags & TTM_PAGE_FLAG_DMA32)
+		gfp_flags = GFP_USER | GFP_DMA32;
+	else
+		gfp_flags = GFP_HIGHUSER;
+	if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
+		gfp_flags |= __GFP_ZERO;
+
+	if (huge) {
+		gfp_flags |= GFP_TRANSHUGE;
+		gfp_flags &= ~__GFP_MOVABLE;
+	}
+
+	return gfp_flags;
+}
+
 /*
  * On success pages list will hold count number of correctly
  * cached pages. On failure will hold the negative return value (-ENOMEM, etc).
@@ -868,6 +926,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 {
 	struct ttm_tt *ttm = &ttm_dma->ttm;
 	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
+	unsigned long num_pages = ttm->num_pages;
 	struct dma_pool *pool;
 	enum pool_type type;
 	unsigned i;
@@ -876,26 +935,61 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	if (ttm->state != tt_unpopulated)
 		return 0;
 
+	INIT_LIST_HEAD(&ttm_dma->pages_list);
+	i = 0;
+
 	type = ttm_to_type(ttm->page_flags, ttm->caching_state);
-	pool = ttm_dma_find_pool(dev, type);
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	if (ttm->page_flags & TTM_PAGE_FLAG_DMA32)
+		goto skip_huge;
+
+	pool = ttm_dma_find_pool(dev, type | IS_HUGE);
 	if (!pool) {
-		gfp_t gfp_flags;
+		gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, true);
 
-		if (ttm->page_flags & TTM_PAGE_FLAG_DMA32)
-			gfp_flags = GFP_USER | GFP_DMA32;
-		else
-			gfp_flags = GFP_HIGHUSER;
-		if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
-			gfp_flags |= __GFP_ZERO;
+		pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE);
+		if (IS_ERR_OR_NULL(pool))
+			goto skip_huge;
+	}
 
-		pool = ttm_dma_pool_init(dev, gfp_flags, type);
-		if (IS_ERR_OR_NULL(pool)) {
+	while (num_pages >= HPAGE_PMD_NR) {
+		unsigned j;
+
+		ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
+		if (ret != 0)
+			break;
+
+		ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
+						pool->size);
+		if (unlikely(ret != 0)) {
+			ttm_dma_unpopulate(ttm_dma, dev);
 			return -ENOMEM;
 		}
+
+		for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) {
+			ttm->pages[j] = ttm->pages[j - 1] + 1;
+			ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] +
+				PAGE_SIZE;
+		}
+
+		i += HPAGE_PMD_NR;
+		num_pages -= HPAGE_PMD_NR;
 	}
 
-	INIT_LIST_HEAD(&ttm_dma->pages_list);
-	for (i = 0; i < ttm->num_pages; ++i) {
+skip_huge:
+#endif
+
+	pool = ttm_dma_find_pool(dev, type);
+	if (!pool) {
+		gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, false);
+
+		pool = ttm_dma_pool_init(dev, gfp_flags, type);
+		if (IS_ERR_OR_NULL(pool))
+			return -ENOMEM;
+	}
+
+	while (num_pages) {
 		ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
 		if (ret != 0) {
 			ttm_dma_unpopulate(ttm_dma, dev);
@@ -908,6 +1002,9 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 			ttm_dma_unpopulate(ttm_dma, dev);
 			return -ENOMEM;
 		}
+
+		++i;
+		--num_pages;
 	}
 
 	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
@@ -931,10 +1028,33 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	struct dma_page *d_page, *next;
 	enum pool_type type;
 	bool is_cached = false;
-	unsigned count = 0, i, npages = 0;
+	unsigned count, i, npages = 0;
 	unsigned long irq_flags;
 
 	type = ttm_to_type(ttm->page_flags, ttm->caching_state);
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	pool = ttm_dma_find_pool(dev, type | IS_HUGE);
+	if (pool) {
+		count = 0;
+		list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list,
+					 page_list) {
+			if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL))
+				continue;
+
+			count++;
+			ttm_mem_global_free_page(ttm->glob->mem_glob,
+						 d_page->p, pool->size);
+			ttm_dma_page_put(pool, d_page);
+		}
+
+		spin_lock_irqsave(&pool->lock, irq_flags);
+		pool->npages_in_use -= count;
+		pool->nfrees += count;
+		spin_unlock_irqrestore(&pool->lock, irq_flags);
+	}
+#endif
+
 	pool = ttm_dma_find_pool(dev, type);
 	if (!pool)
 		return;
@@ -943,6 +1063,7 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 		     ttm_to_type(ttm->page_flags, tt_cached)) == pool);
 
 	/* make sure pages array match list and count number of pages */
+	count = 0;
 	list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) {
 		ttm->pages[count] = d_page->p;
 		count++;