ppc64: hugetlb fixes for LPAR and numa hugetlb support

arch/ppc64/kernel/pSeries_lpar.c

@@ -36,18 +36,6 @@
 #include <asm/tlb.h>
 #include <asm/hvcall.h>
 
-
-long plpar_pte_enter(unsigned long flags,
-		     unsigned long ptex,
-		     unsigned long new_pteh, unsigned long new_ptel,
-		     unsigned long *old_pteh_ret, unsigned long *old_ptel_ret)
-{
-	unsigned long dummy, ret;
-	ret = plpar_hcall(H_ENTER, flags, ptex, new_pteh, new_ptel,
-			   old_pteh_ret, old_ptel_ret, &dummy);
-	return(ret);
-}
-
 long plpar_pte_remove(unsigned long flags,
 		      unsigned long ptex,
 		      unsigned long avpn,
@@ -83,7 +71,6 @@ long plpar_tce_get(unsigned long liobn,
 			   tce_ret, &dummy, &dummy);
 }
 
-
 long plpar_tce_put(unsigned long liobn,
 		   unsigned long ioba,
 		   unsigned long tceval)
@@ -104,10 +91,9 @@ long plpar_put_term_char(unsigned long termno,
 			 unsigned long len,
 			 const char *buffer)
 {
-	unsigned long dummy;
 	unsigned long *lbuf = (unsigned long *)buffer;  /* ToDo: alignment? */
-	return plpar_hcall(H_PUT_TERM_CHAR, termno, len,
-			   lbuf[0], lbuf[1], &dummy, &dummy, &dummy);
+	return plpar_hcall_norets(H_PUT_TERM_CHAR, termno, len, lbuf[0],
+				  lbuf[1]);
 }
 
 static void tce_build_pSeriesLP(struct TceTable *tbl, long tcenum, 
@@ -287,12 +273,11 @@ int hvc_get_chars(int index, char *buf, int count)
 
 int hvc_put_chars(int index, const char *buf, int count)
 {
-	unsigned long dummy;
 	unsigned long *lbuf = (unsigned long *) buf;
 	long ret;
 
-	ret = plpar_hcall(H_PUT_TERM_CHAR, index, count, lbuf[0], lbuf[1],
-			  &dummy, &dummy, &dummy);
+	ret = plpar_hcall_norets(H_PUT_TERM_CHAR, index, count, lbuf[0],
+				 lbuf[1]);
 	if (ret == H_Success)
 		return count;
 	if (ret == H_Busy)
@@ -318,7 +303,6 @@ int hvc_count(int *start_termno)
 
 
 
-
 long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 			      unsigned long va, unsigned long prpn,
 			      int secondary, unsigned long hpteflags,
@@ -329,6 +313,7 @@ long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 	unsigned long flags;
 	unsigned long slot;
 	HPTE lhpte;
+	unsigned long dummy0, dummy1;
 
 	/* Fill in the local HPTE with absolute rpn, avpn and flags */
 	lhpte.dw1.dword1      = 0;
@@ -348,7 +333,6 @@ long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 
 	/* Now fill in the actual HPTE */
 	/* Set CEC cookie to 0         */
-	/* Large page = 0              */
 	/* Zero page = 0               */
 	/* I-cache Invalidate = 0      */
 	/* I-cache synchronize = 0     */
@@ -359,19 +343,8 @@ long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 	if (hpteflags & (_PAGE_GUARDED|_PAGE_NO_CACHE))
 		lhpte.dw1.flags.flags &= ~_PAGE_COHERENT;
 
-	__asm__ __volatile__ (
-		H_ENTER_r3
-		"mr    4, %2\n"
-                "mr    5, %3\n"
-                "mr    6, %4\n"
-                "mr    7, %5\n"
-                HSC    
-                "mr    %0, 3\n"
-                "mr    %1, 4\n"
-		: "=r" (lpar_rc), "=r" (slot)
-		: "r" (flags), "r" (hpte_group), "r" (lhpte.dw0.dword0),
-		"r" (lhpte.dw1.dword1)
-		: "r3", "r4", "r5", "r6", "r7", "cc");
+	lpar_rc = plpar_hcall(H_ENTER, flags, hpte_group, lhpte.dw0.dword0,
+			      lhpte.dw1.dword1, &slot, &dummy0, &dummy1);
 
 	if (lpar_rc == H_PTEG_Full)
 		return -1;

arch/ppc64/mm/hugetlbpage.c

@@ -37,7 +37,56 @@ static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
 
 static int htlbpage_free; /* = 0 */
 static int htlbpage_total; /* = 0 */
-static LIST_HEAD(htlbpage_freelist);
+static struct list_head hugepage_freelists[MAX_NUMNODES];
+
+static void enqueue_huge_page(struct page *page)
+{
+	list_add(&page->list,
+		&hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
+}
+
+/* XXX make this a sysctl */
+unsigned long largepage_roundrobin = 1;
+
+static struct page *dequeue_huge_page(void)
+{
+	static int nid = 0;
+	struct page *page = NULL;
+	int i;
+
+	if (!largepage_roundrobin)
+		nid = numa_node_id();
+
+	for (i = 0; i < numnodes; i++) {
+		if (!list_empty(&hugepage_freelists[nid]))
+			break;
+		nid = (nid + 1) % numnodes;
+	}
+
+	if (!list_empty(&hugepage_freelists[nid])) {
+		page = list_entry(hugepage_freelists[nid].next, struct page, list);
+		list_del(&page->list);
+	}
+
+	if (largepage_roundrobin)
+		nid = (nid + 1) % numnodes;
+
+	return page;
+}
+
+static struct page *alloc_fresh_huge_page(void)
+{
+	static int nid = 0;
+	struct page *page;
+
+	page = alloc_pages_node(nid, GFP_HIGHUSER, HUGETLB_PAGE_ORDER);
+	if (!page)
+		return NULL;
+
+	nid = page_zone(page)->zone_pgdat->node_id;
+	nid = (nid + 1) % numnodes;
+	return page;
+}
 
 /* HugePTE layout:
  *
@@ -103,13 +152,12 @@ static struct page *alloc_hugetlb_page(void)
 	struct page *page;
 
 	spin_lock(&htlbpage_lock);
-	if (list_empty(&htlbpage_freelist)) {
+	page = dequeue_huge_page();
+	if (!page) {
 		spin_unlock(&htlbpage_lock);
 		return NULL;
 	}
 
-	page = list_entry(htlbpage_freelist.next, struct page, list);
-	list_del(&page->list);
 	htlbpage_free--;
 	spin_unlock(&htlbpage_lock);
 	set_page_count(page, 1);
@@ -365,7 +413,7 @@ static void free_huge_page(struct page *page)
 	INIT_LIST_HEAD(&page->list);
 
 	spin_lock(&htlbpage_lock);
-	list_add(&page->list, &htlbpage_freelist);
+	enqueue_huge_page(page);
 	htlbpage_free++;
 	spin_unlock(&htlbpage_lock);
 }
@@ -604,9 +652,11 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
 	if (!in_hugepage_area(mm->context, ea))
 		return -1;
 
+	ea &= ~(HPAGE_SIZE-1);
+
 	/* We have to find the first hugepte in the batch, since
 	 * that's the one that will store the HPTE flags */
-	ptep = hugepte_offset(mm, ea & ~(HPAGE_SIZE-1));
+	ptep = hugepte_offset(mm, ea);
 
 	/* Search the Linux page table for a match with va */
 	va = (vsid << 28) | (ea & 0x0fffffff);
@@ -675,6 +725,10 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
 		hugepte_val(new_pte) &= ~_HUGEPAGE_HPTEFLAGS;
 		hugepte_val(new_pte) |= _HUGEPAGE_HASHPTE;
 
+		/* Add in WIMG bits */
+		/* XXX We should store these in the pte */
+		hpteflags |= _PAGE_COHERENT;
+
 		slot = ppc_md.hpte_insert(hpte_group, va, prpn, 0,
 					  hpteflags, 0, 1);
 
@@ -695,7 +749,7 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access,
 		}
 
 		if (unlikely(slot == -2))
-			panic("hash_page: pte_insert failed\n");
+			panic("hash_huge_page: pte_insert failed\n");
 
 		hugepte_val(new_pte) |= (slot<<5) & _HUGEPAGE_GROUP_IX;
 
@@ -768,11 +822,11 @@ int set_hugetlb_mem_size(int count)
 		return htlbpage_total;
 	if (lcount > 0) {	/* Increase the mem size. */
 		while (lcount--) {
-			page = alloc_pages(__GFP_HIGHMEM, HUGETLB_PAGE_ORDER);
+			page = alloc_fresh_huge_page();
 			if (page == NULL)
 				break;
 			spin_lock(&htlbpage_lock);
-			list_add(&page->list, &htlbpage_freelist);
+			enqueue_huge_page(page);
 			htlbpage_free++;
 			htlbpage_total++;
 			spin_unlock(&htlbpage_lock);
@@ -813,12 +867,15 @@ static int __init hugetlb_init(void)
 	struct page *page;
 
 	if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) {
+		for (i = 0; i < MAX_NUMNODES; ++i)
+			INIT_LIST_HEAD(&hugepage_freelists[i]);
+
 		for (i = 0; i < htlbpage_max; ++i) {
-			page = alloc_pages(__GFP_HIGHMEM, HUGETLB_PAGE_ORDER);
+			page = alloc_fresh_huge_page();
 			if (!page)
 				break;
 			spin_lock(&htlbpage_lock);
-			list_add(&page->list, &htlbpage_freelist);
+			enqueue_huge_page(page);
 			spin_unlock(&htlbpage_lock);
 		}
 		htlbpage_max = htlbpage_free = htlbpage_total = i;
@@ -827,7 +884,7 @@ static int __init hugetlb_init(void)
 		htlbpage_max = 0;
 		printk("CPU does not support HugeTLB\n");
 	}
-	
+
 	return 0;
 }
 module_init(hugetlb_init);

include/asm-ppc64/hvcall.h

@@ -59,9 +59,6 @@
 #define H_XIRR			0x74
 #define H_PERFMON		0x7c
 
-#define HSC			".long 0x44000022\n"
-#define H_ENTER_r3		"li	3, 0x08\n"
-
 /* plpar_hcall() -- Generic call interface using above opcodes
  *
  * The actual call interface is a hypervisor call instruction with