Merge samba.org:/scratch/anton/linux-2.5

into samba.org:/scratch/anton/linux-2.5_ppc64

arch/ppc64/config.in

@@ -23,6 +23,10 @@ if [ "$CONFIG_SMP" = "y" ]; then
   bool '  Distribute interrupts on all CPUs by default' CONFIG_IRQ_ALL_CPUS
   if [ "$CONFIG_PPC_PSERIES" = "y" ]; then
     bool '  Hardware multithreading' CONFIG_HMT
+    bool '  Discontiguous Memory Support' CONFIG_DISCONTIGMEM
+    if [ "$CONFIG_DISCONTIGMEM" = "y" ]; then
+      bool '    NUMA support' CONFIG_NUMA
+    fi
   fi
 fi
 define_bool CONFIG_PREEMPT n

arch/ppc64/kernel/eeh.c

@@ -59,15 +59,14 @@ unsigned long eeh_token(unsigned long phb, unsigned long bus, unsigned long devf
 	return ((IO_UNMAPPED_REGION_ID << 60) | (phb << 48UL) | ((bus & 0xff) << 40UL) | (devfn << 32UL) | (offset & 0xffffffff));
 }
 
-
-
-int eeh_get_state(unsigned long ea) {
+int eeh_get_state(unsigned long ea)
+{
 	return 0;
 }
 
-
 /* Check for an eeh failure at the given token address.
- * The given value has been read and it should be 1's (0xff, 0xffff or 0xffffffff).
+ * The given value has been read and it should be 1's (0xff, 0xffff or
+ * 0xffffffff).
  *
  * Probe to determine if an error actually occurred.  If not return val.
  * Otherwise panic.
@@ -113,7 +112,8 @@ unsigned long eeh_check_failure(void *token, unsigned long val)
 	return val;	/* good case */
 }
 
-void eeh_init(void) {
+void eeh_init(void)
+{
 	extern char cmd_line[];	/* Very early cmd line parse.  Cheap, but works. */
 	char *eeh_force_off = strstr(cmd_line, "eeh-force-off");
 	char *eeh_force_on = strstr(cmd_line, "eeh-force-on");
@@ -121,7 +121,7 @@ void eeh_init(void) {
 	ibm_set_eeh_option = rtas_token("ibm,set-eeh-option");
 	ibm_set_slot_reset = rtas_token("ibm,set-slot-reset");
 	ibm_read_slot_reset_state = rtas_token("ibm,read-slot-reset-state");
-	if (ibm_set_eeh_option != RTAS_UNKNOWN_SERVICE && naca->platform == PLATFORM_PSERIES_LPAR)
+	if (ibm_set_eeh_option != RTAS_UNKNOWN_SERVICE)
 		eeh_implemented = 1;
 
 	if (eeh_force_off > eeh_force_on) {
@@ -334,6 +334,7 @@ static int __init eehoff_parm(char *str)
 {
 	return eeh_parm(str, 0);
 }
+
 static int __init eehon_parm(char *str)
 {
 	return eeh_parm(str, 1);

arch/ppc64/kernel/entry.S

@@ -353,7 +353,7 @@ recheck:
 	li	r4,0
 	ori	r4,r4,MSR_EE|MSR_RI
 	andc	r10,r10,r4	/* clear MSR_EE and MSR_RI */
-	mtmsrd	r10		/* Update machine state */
+	mtmsrd	r10,1		/* Update machine state */
 
 #ifdef CONFIG_PPC_ISERIES
 #error fix iSeries soft disable
@@ -411,7 +411,7 @@ restore:
 do_work:
 	/* Enable interrupts */
 	ori	r10,r10,MSR_EE|MSR_RI
-	mtmsrd	r10
+	mtmsrd	r10,1
 
 	andi.	r0,r3,_TIF_NEED_RESCHED
 	beq	1f

arch/ppc64/kernel/head.S

@@ -575,7 +575,8 @@ stab_bolted_user_return:
 	bl	.do_stab_SI
 	b	1f
 
-2:	bl	.do_hash_page_DSI 	/* Try to handle as hpte fault */
+2:	li	r5,0x300
+	bl	.do_hash_page_DSI 	/* Try to handle as hpte fault */
 1:
 	ld      r4,_DAR(r1)
 	ld      r5,_DSISR(r1)
@@ -627,9 +628,8 @@ InstructionAccess_common:
 	bl	.do_stab_SI
 	b	1f
 
-2:	andis.	r0,r23,0x4000		/* no pte found? */
-	beq	1f			/* if so, try to put a PTE */
-	mr	r3,r22			/* into the hash table */
+2:	mr	r3,r22
+	li	r5,0x400
 	bl	.do_hash_page_ISI	/* Try to handle as hpte fault */
 1:
 	mr	r4,r22
@@ -804,6 +804,7 @@ _GLOBAL(do_hash_page_DSI)
 	/*
 	 * r3 contains the faulting address
 	 * r4 contains the required access permissions
+	 * r5 contains the trap number
 	 *
 	 * at return r3 = 0 for success
 	 */
@@ -1119,7 +1120,7 @@ _GLOBAL(save_remaining_regs)
 	rldimi	r22,r20,15,48		/* Insert desired EE value */
 #endif
 
-	mtmsrd  r22
+	mtmsrd  r22,1
 	blr
 
 

arch/ppc64/kernel/htab.c

@@ -45,9 +45,8 @@
 #include <asm/lmb.h>
 #include <asm/abs_addr.h>
 #include <asm/tlbflush.h>
-#ifdef CONFIG_PPC_EEH
 #include <asm/eeh.h>
-#endif
+#include <asm/tlb.h>
 
 /*
  * Note:  pte   --> Linux PTE
@@ -196,7 +195,7 @@ static inline unsigned long computeHptePP(unsigned long pte)
  * to be valid via Linux page tables, return 1. If handled return 0
  */
 int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
-		pte_t *ptep)
+		pte_t *ptep, unsigned long trap)
 {
 	unsigned long va, vpn;
 	unsigned long newpp, prpn;
@@ -245,6 +244,24 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
 
 	newpp = computeHptePP(pte_val(new_pte));
 
+#define PPC64_HWNOEXEC (1 << 2)
+
+	/* We do lazy icache flushing on POWER4 */
+	if (__is_processor(PV_POWER4) && pfn_valid(pte_pfn(new_pte))) {
+		struct page *page = pte_page(new_pte);
+
+		/* page is dirty */
+		if (!PageReserved(page) &&
+		    !test_bit(PG_arch_1, &page->flags)) {
+			if (trap == 0x400) {
+				__flush_dcache_icache(page_address(page));
+				set_bit(PG_arch_1, &page->flags);
+			} else {
+				newpp |= PPC64_HWNOEXEC;
+			}
+		}
+	}
+
 	/* Check if pte already has an hpte (case 2) */
 	if (pte_val(old_pte) & _PAGE_HASHPTE) {
 		/* There MIGHT be an HPTE for this pte */
@@ -318,7 +335,7 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
 	return 0;
 }
 
-int hash_page(unsigned long ea, unsigned long access)
+int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 {
 	void *pgdir;
 	unsigned long vsid;
@@ -346,13 +363,11 @@ int hash_page(unsigned long ea, unsigned long access)
 		mm = &init_mm;
 		vsid = get_kernel_vsid(ea);
 		break;
-#ifdef CONFIG_PPC_EEH
 	case IO_UNMAPPED_REGION_ID:
 		udbg_printf("EEH Error ea = 0x%lx\n", ea);
 		PPCDBG_ENTER_DEBUGGER();
 		panic("EEH Error ea = 0x%lx\n", ea);
 		break;
-#endif
 	case KERNEL_REGION_ID:
 		/*
 		 * As htab_initialize is now, we shouldn't ever get here since
@@ -379,7 +394,7 @@ int hash_page(unsigned long ea, unsigned long access)
 	 */
 	spin_lock(&mm->page_table_lock);
 	ptep = find_linux_pte(pgdir, ea);
-	ret = __hash_page(ea, access, vsid, ptep);
+	ret = __hash_page(ea, access, vsid, ptep, trap);
 	spin_unlock(&mm->page_table_lock);
 
 	return ret;
@@ -419,12 +434,11 @@ void flush_hash_range(unsigned long context, unsigned long number, int local)
 		ppc_md.flush_hash_range(context, number, local);
 	} else {
 		int i;
-		struct tlb_batch_data *ptes =
-			&tlb_batch_array[smp_processor_id()][0];
+		struct ppc64_tlb_batch *batch =
+			&ppc64_tlb_batch[smp_processor_id()];
 
-		for (i = 0; i < number; i++) {
-			flush_hash_page(context, ptes->addr, ptes->pte, local);
-			ptes++;
-		}
+		for (i = 0; i < number; i++)
+			flush_hash_page(context, batch->addr[i], batch->pte[i],
+					local);
 	}
 }

arch/ppc64/kernel/irq.c

@@ -474,6 +474,8 @@ void ppc_irq_dispatch_handler(struct pt_regs *regs, int irq)
 	int cpu = smp_processor_id();
 	irq_desc_t *desc = irq_desc + irq;
 
+	/* XXX This causes bad performance and lockups on XICS - Anton */
+	if (naca->interrupt_controller == IC_OPEN_PIC)
 		balance_irq(irq);
 
 	kstat.irqs[cpu][irq]++;

arch/ppc64/kernel/pSeries_htab.c

@@ -20,6 +20,7 @@
 #include <asm/mmu_context.h>
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
+#include <asm/tlb.h>
 
 /*
  * Create a pte. Used during initialization only.
@@ -214,7 +215,7 @@ static inline void set_pp_bit(unsigned long pp, HPTE *addr)
 
 	__asm__ __volatile__(
 	"1:	ldarx	%0,0,%3\n\
-		rldimi	%0,%2,0,62\n\
+		rldimi	%0,%2,0,61\n\
 		stdcx.	%0,0,%3\n\
 		bne	1b"
 	: "=&r" (old), "=m" (*p)
@@ -265,8 +266,6 @@ static long pSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
 	unsigned long vpn, avpn;
 	unsigned long flags;
 
-	udbg_printf("updatepp\n");
-
 	if (large)
 		vpn = va >> LARGE_PAGE_SHIFT;
 	else
@@ -372,31 +371,32 @@ static void pSeries_flush_hash_range(unsigned long context,
 {
 	unsigned long vsid, vpn, va, hash, secondary, slot, flags, avpn;
 	int i, j;
-	unsigned long va_array[MAX_BATCH_FLUSH];
 	HPTE *hptep;
 	Hpte_dword0 dw0;
-	struct tlb_batch_data *ptes = &tlb_batch_array[smp_processor_id()][0];
+	struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[smp_processor_id()];
 	/* XXX fix for large ptes */
 	unsigned long large = 0;
+
 	j = 0;
 	for (i = 0; i < number; i++) {
-		if ((ptes->addr >= USER_START) && (ptes->addr <= USER_END))
-			vsid = get_vsid(context, ptes->addr);
+		if ((batch->addr[i] >= USER_START) &&
+		    (batch->addr[i] <= USER_END))
+			vsid = get_vsid(context, batch->addr[i]);
 		else
-			vsid = get_kernel_vsid(ptes->addr);
+			vsid = get_kernel_vsid(batch->addr[i]);
 
-		va = (vsid << 28) | (ptes->addr & 0x0fffffff);
-		va_array[j] = va;
+		va = (vsid << 28) | (batch->addr[i] & 0x0fffffff);
+		batch->vaddr[j] = va;
 		if (large)
 			vpn = va >> LARGE_PAGE_SHIFT;
 		else
 			vpn = va >> PAGE_SHIFT;
 		hash = hpt_hash(vpn, large);
-		secondary = (pte_val(ptes->pte) & _PAGE_SECONDARY) >> 15;
+		secondary = (pte_val(batch->pte[i]) & _PAGE_SECONDARY) >> 15;
 		if (secondary)
 			hash = ~hash;
 		slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
-		slot += (pte_val(ptes->pte) & _PAGE_GROUP_IX) >> 12;
+		slot += (pte_val(batch->pte[i]) & _PAGE_GROUP_IX) >> 12;
 
 		hptep = htab_data.htab + slot;
 		avpn = vpn >> 11;
@@ -405,8 +405,6 @@ static void pSeries_flush_hash_range(unsigned long context,
 
 		dw0 = hptep->dw0.dw0;
 
-		ptes++;
-
 		if ((dw0.avpn != avpn) || !dw0.v) {
 			pSeries_unlock_hpte(hptep);
 			udbg_printf("invalidate missed\n");
@@ -426,7 +424,7 @@ static void pSeries_flush_hash_range(unsigned long context,
 			asm volatile("\n\
 			clrldi  %0,%0,16\n\
 			tlbiel   %0"
-			: : "r" (va_array[i]) : "memory" );
+			: : "r" (batch->vaddr[i]) : "memory" );
 		}
 
 		asm volatile("ptesync":::"memory");
@@ -440,7 +438,7 @@ static void pSeries_flush_hash_range(unsigned long context,
 			asm volatile("\n\
 			clrldi  %0,%0,16\n\
 			tlbie   %0"
-			: : "r" (va_array[i]) : "memory" );
+			: : "r" (batch->vaddr[i]) : "memory" );
 		}
 
 		asm volatile("eieio; tlbsync; ptesync":::"memory");

arch/ppc64/kernel/pSeries_lpar.c

@@ -33,6 +33,7 @@
 #include <linux/pci.h>
 #include <asm/naca.h>
 #include <asm/tlbflush.h>
+#include <asm/tlb.h>
 
 /* Status return values */
 #define H_Success	0
@@ -646,11 +647,9 @@ static long pSeries_lpar_hpte_updatepp(unsigned long slot, unsigned long newpp,
 {
 	unsigned long lpar_rc;
 	unsigned long flags;
-	flags = (newpp & 3) | H_AVPN;
+	flags = (newpp & 7) | H_AVPN;
 	unsigned long vpn = va >> PAGE_SHIFT;
 
-	udbg_printf("updatepp\n");
-
 	lpar_rc = plpar_pte_protect(flags, slot, (vpn >> 4) & ~0x7fUL);
 
 	if (lpar_rc == H_Not_Found) {
@@ -775,15 +774,14 @@ void pSeries_lpar_flush_hash_range(unsigned long context, unsigned long number,
 				   int local)
 {
 	int i;
-	struct tlb_batch_data *ptes =
-		&tlb_batch_array[smp_processor_id()][0];
 	unsigned long flags;
+	struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[smp_processor_id()];
 
 	spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
-	for (i = 0; i < number; i++) {
-		flush_hash_page(context, ptes->addr, ptes->pte, local);
-		ptes++;
-	}
+
+	for (i = 0; i < number; i++)
+		flush_hash_page(context, batch->addr[i], batch->pte[i], local);
+
 	spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags);
 }
 

arch/ppc64/kernel/proc_pmc.c

@@ -100,9 +100,6 @@ void proc_ppc64_init(void)
 	if (!proc_ppc64_root) return;
 	spin_unlock(&proc_ppc64_lock);
 
-#ifdef CONFIG_PPC_EEH
-	eeh_init_proc(proc_ppc64_root);
-#endif
 
 	proc_ppc64_pmc_root = proc_mkdir("pmc", proc_ppc64_root);
 

arch/ppc64/kernel/smp.c

@@ -595,12 +595,8 @@ void __init smp_boot_cpus(void)
 	}
 
 	/*
-	 * XXX very rough. On POWER4 we optimise tlb flushes for
-	 * tasks that only run on one cpu so we increase decay ticks.
+	 * XXX very rough. 
 	 */
-	if (__is_processor(PV_POWER4))
-		cache_decay_ticks = HZ/50;
-	else
 	cache_decay_ticks = HZ/100;
 
 	/* Probe arch for CPUs */

arch/ppc64/mm/init.c

 /*
- *  
- *
  *  PowerPC version 
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *
@@ -62,8 +60,6 @@
 
 #include <asm/ppcdebug.h>
 
-#define	PGTOKB(pages)	(((pages) * PAGE_SIZE) >> 10)
-
 #ifdef CONFIG_PPC_ISERIES
 #include <asm/iSeries/iSeries_dma.h>
 #endif
@@ -78,13 +74,10 @@ extern pgd_t swapper_pg_dir[];
 extern char __init_begin, __init_end;
 extern char __chrp_begin, __chrp_end;
 extern char __openfirmware_begin, __openfirmware_end;
-extern struct _of_tce_table of_tce_table[];
 extern char _start[], _end[];
 extern char _stext[], etext[];
 extern struct task_struct *current_set[NR_CPUS];
 
-void mm_init_ppc64(void);
-
 extern pgd_t ioremap_dir[];
 pgd_t * ioremap_pgd = (pgd_t *)&ioremap_dir;
 
@@ -120,13 +113,10 @@ void show_mem(void)
 			reserved++;
 		else if (PageSwapCache(mem_map+i))
 			cached++;
-		else if (!atomic_read(&mem_map[i].count))
-			free++;
-		else
-			shared += atomic_read(&mem_map[i].count) - 1;
+		else if (page_count(mem_map+i))
+			shared += page_count(mem_map+i) - 1;
 	}
 	printk("%d pages of RAM\n",total);
-	printk("%d free pages\n",free);
 	printk("%d reserved pages\n",reserved);
 	printk("%d pages shared\n",shared);
 	printk("%d pages swap cached\n",cached);
@@ -302,7 +292,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 	}
 }
 
-struct tlb_batch_data tlb_batch_array[NR_CPUS][MAX_BATCH_FLUSH];
+struct ppc64_tlb_batch ppc64_tlb_batch[NR_CPUS];
 
 void
 __flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
@@ -312,81 +302,69 @@ __flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
 	pte_t *ptep;
 	pte_t pte;
 	unsigned long pgd_end, pmd_end;
-	unsigned long context;
-	int i = 0;
-	struct tlb_batch_data *ptes = &tlb_batch_array[smp_processor_id()][0];
+	unsigned long context = 0;
+	struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[smp_processor_id()];
+	unsigned long i = 0;
 	int local = 0;
 
-	if ( start >= end )
-		panic("flush_tlb_range: start (%016lx) greater than end (%016lx)\n", start, end );
-
-	if ( REGION_ID(start) != REGION_ID(end) )
-		panic("flush_tlb_range: start (%016lx) and end (%016lx) not in same region\n", start, end );
-	
-	context = 0;
-
-	switch( REGION_ID(start) ) {
+	switch(REGION_ID(start)) {
 	case VMALLOC_REGION_ID:
-		pgd = pgd_offset_k( start );
+		pgd = pgd_offset_k(start);
 		break;
 	case IO_REGION_ID:
-		pgd = pgd_offset_i( start );
+		pgd = pgd_offset_i(start);
 		break;
 	case USER_REGION_ID:
-		pgd = pgd_offset( mm, start );
+		pgd = pgd_offset(mm, start);
 		context = mm->context;
 
 		/* XXX are there races with checking cpu_vm_mask? - Anton */
-		if (mm->cpu_vm_mask == (1 << smp_processor_id())) {
+		if (mm->cpu_vm_mask == (1 << smp_processor_id()))
 			local = 1;
-		}
 
 		break;
 	default:
 		panic("flush_tlb_range: invalid region for start (%016lx) and end (%016lx)\n", start, end);
-	
 	}
 
 	do {
 		pgd_end = (start + PGDIR_SIZE) & PGDIR_MASK;
-		if ( pgd_end > end ) 
+		if (pgd_end > end)
 			pgd_end = end;
-		if ( !pgd_none( *pgd ) ) {
-			pmd = pmd_offset( pgd, start );
+		if (!pgd_none(*pgd)) {
+			pmd = pmd_offset(pgd, start);
 			do {
-				pmd_end = ( start + PMD_SIZE ) & PMD_MASK;
-				if ( pmd_end > end )
+				pmd_end = (start + PMD_SIZE) & PMD_MASK;
+				if (pmd_end > end)
 					pmd_end = end;
-				if ( !pmd_none( *pmd ) ) {
-					ptep = pte_offset_kernel( pmd, start );
+				if (!pmd_none(*pmd)) {
+					ptep = pte_offset_kernel(pmd, start);
 					do {
-						if ( pte_val(*ptep) & _PAGE_HASHPTE ) {
+						if (pte_val(*ptep) & _PAGE_HASHPTE) {
 							pte = __pte(pte_update(ptep, _PAGE_HPTEFLAGS, 0));
-							if ( pte_val(pte) & _PAGE_HASHPTE ) {
-								ptes->pte = pte;
-								ptes->addr = start;
-								ptes++;
+							if (pte_val(pte) & _PAGE_HASHPTE) {								
+								batch->pte[i] = pte;
+								batch->addr[i] = start;
 								i++;
-								if (i == MAX_BATCH_FLUSH) {
-									flush_hash_range(context, MAX_BATCH_FLUSH, local);
+								if (i == PPC64_TLB_BATCH_NR) {
+									flush_hash_range(context, i, local);
 									i = 0;
-									ptes = &tlb_batch_array[smp_processor_id()][0];
 								}
 							}
 						}
 						start += PAGE_SIZE;
 						++ptep;
-					} while ( start < pmd_end );
-				}
-				else
+					} while (start < pmd_end);
+				} else {
 					start = pmd_end;
-				++pmd;
-			} while ( start < pgd_end );
 				}
-		else
+				++pmd;
+			} while (start < pgd_end);
+		} else {
 			start = pgd_end;
+		}
 		++pgd;
-	} while ( start < end );
+	} while (start < end);
 
 	if (i)
 		flush_hash_range(context, i, local);
@@ -463,7 +441,6 @@ void __init do_init_bootmem(void)
 	unsigned long start, bootmap_pages;
 	unsigned long total_pages = lmb_end_of_DRAM() >> PAGE_SHIFT;
 
-	PPCDBG(PPCDBG_MMINIT, "do_init_bootmem: start\n");
 	/*
 	 * Find an area to use for the bootmem bitmap.  Calculate the size of
 	 * bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
@@ -472,21 +449,16 @@ void __init do_init_bootmem(void)
 	bootmap_pages = bootmem_bootmap_pages(total_pages);
 
 	start = (unsigned long)__a2p(lmb_alloc(bootmap_pages<<PAGE_SHIFT, PAGE_SIZE));
-	if( start == 0 ) {
+	if (start == 0) {
 		udbg_printf("do_init_bootmem: failed to allocate a bitmap.\n");
 		udbg_printf("\tbootmap_pages = 0x%lx.\n", bootmap_pages);
 		PPCDBG_ENTER_DEBUGGER(); 
 	}
 
-	PPCDBG(PPCDBG_MMINIT, "\tstart               = 0x%lx\n", start);
-	PPCDBG(PPCDBG_MMINIT, "\tbootmap_pages       = 0x%lx\n", bootmap_pages);
-	PPCDBG(PPCDBG_MMINIT, "\tphysicalMemorySize  = 0x%lx\n", naca->physicalMemorySize);
-
 	boot_mapsize = init_bootmem(start >> PAGE_SHIFT, total_pages);
-	PPCDBG(PPCDBG_MMINIT, "\tboot_mapsize        = 0x%lx\n", boot_mapsize);
 
 	/* add all physical memory to the bootmem map */
-	for (i=0; i < lmb.memory.cnt ;i++) {
+	for (i=0; i < lmb.memory.cnt; i++) {
 		unsigned long physbase, size;
 		unsigned long type = lmb.memory.region[i].type;
 
@@ -497,19 +469,14 @@ void __init do_init_bootmem(void)
 		size = lmb.memory.region[i].size;
 		free_bootmem(physbase, size);
 	}
+
 	/* reserve the sections we're already using */
-	for (i=0; i < lmb.reserved.cnt ;i++) {
+	for (i=0; i < lmb.reserved.cnt; i++) {
 		unsigned long physbase = lmb.reserved.region[i].physbase;
 		unsigned long size = lmb.reserved.region[i].size;
-#if 0 /* PPPBBB */
-		if ( (physbase == 0) && (size < (16<<20)) ) {
-			size = 16 << 20;
-		}
-#endif
+
 		reserve_bootmem(physbase, size);
 	}
-
-	PPCDBG(PPCDBG_MMINIT, "do_init_bootmem: end\n");
 }
 
 /*
@@ -522,7 +489,7 @@ void __init paging_init(void)
 	/*
 	 * All pages are DMA-able so we put them all in the DMA zone.
 	 */
-	zones_size[0] = lmb_end_of_DRAM() >> PAGE_SHIFT;
+	zones_size[ZONE_DMA] = lmb_end_of_DRAM() >> PAGE_SHIFT;
 	for (i = 1; i < MAX_NR_ZONES; i++)
 		zones_size[i] = 0;
 	free_area_init(zones_size);
@@ -554,14 +521,6 @@ void __init mem_init(void)
 
 	totalram_pages += free_all_bootmem();
 
-	ifppcdebug(PPCDBG_MMINIT) {
-		udbg_printf("mem_init: totalram_pages = 0x%lx\n", totalram_pages);
-		udbg_printf("mem_init: va_rtas_base   = 0x%lx\n", va_rtas_base); 
-		udbg_printf("mem_init: va_rtas_end    = 0x%lx\n", PAGE_ALIGN(va_rtas_base+rtas.size)); 
-		udbg_printf("mem_init: pinned start   = 0x%lx\n", __va(0)); 
-		udbg_printf("mem_init: pinned end     = 0x%lx\n", PAGE_ALIGN(klimit)); 
-	}
-
 	if ( sysmap_size )
 		for (addr = (unsigned long)sysmap;
 		     addr < PAGE_ALIGN((unsigned long)sysmap+sysmap_size) ;
@@ -613,6 +572,12 @@ void flush_dcache_page(struct page *page)
 
 void flush_icache_page(struct vm_area_struct *vma, struct page *page)
 {
+	if (__is_processor(PV_POWER4))
+		return;
+
+	if ((vma->vm_flags & VM_EXEC) == 0)
+		return;
+
 	if (page->mapping && !PageReserved(page)
 	    && !test_bit(PG_arch_1, &page->flags)) {
 		__flush_dcache_icache(page_address(page));
@@ -620,14 +585,34 @@ void flush_icache_page(struct vm_area_struct *vma, struct page *page)
 	}
 }
 
-void clear_user_page(void *page, unsigned long vaddr)
+void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
 {
 	clear_page(page);
+
+	/* XXX we shouldnt have to do this, but glibc requires it */
+	if (__is_processor(PV_POWER4))
+		clear_bit(PG_arch_1, &pg->flags);
+	else
+		__flush_dcache_icache(page);
 }
 
-void copy_user_page(void *vto, void *vfrom, unsigned long vaddr)
+void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
+		    struct page *pg)
 {
 	copy_page(vto, vfrom);
+
+	/*
+	 * Unfortunately we havent always marked our GOT and PLT sections
+	 * as executable, so we need to flush all file regions - Anton
+	 */
+#if 0
+	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
+		return;
+#endif
+
+	if (__is_processor(PV_POWER4))
+		clear_bit(PG_arch_1, &pg->flags);
+	else
 		__flush_dcache_icache(vto);
 }
 
@@ -642,7 +627,7 @@ void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
 
 extern pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea);
 int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
-		pte_t *ptep);
+		pte_t *ptep, unsigned long trap);
 
 /*
  * This is called at the end of handling a user page fault, when the
@@ -670,5 +655,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long ea,
 	ptep = find_linux_pte(pgdir, ea);
 	vsid = get_vsid(vma->vm_mm->context, ea);
 
-	__hash_page(ea, pte_val(pte) & (_PAGE_USER|_PAGE_RW), vsid, ptep);
+	__hash_page(ea, pte_val(pte) & (_PAGE_USER|_PAGE_RW), vsid, ptep,
+		    0x300);
 }

arch/ppc64/xmon/xmon.c

@@ -123,11 +123,7 @@ static void mem_translate(void);
 static void mem_check(void);
 static void mem_find_real(void);
 static void mem_find_vsid(void);
-static void mem_check_pagetable_vsids (void);
 
-static void mem_map_check_slab(void);
-static void mem_map_lock_pages(void);
-static void mem_check_dup_rpn (void);
 static void debug_trace(void);
 
 extern int print_insn_big_powerpc(FILE *, unsigned long, unsigned long);
@@ -642,27 +638,15 @@ cmds(struct pt_regs *excp)
 			case 'c':
 				mem_check();
 				break;
-			case 'j':
-				mem_map_check_slab();
-				break;
 			case 'f':
 				mem_find_real();
 				break;
 			case 'e':
 				mem_find_vsid();
 				break;
-			case 'r':
-				mem_check_dup_rpn();
-				break;           
 			case 'i':
 				show_mem();
 				break;
-			case 'o':
-				mem_check_pagetable_vsids ();
-				break;
-			case 'q':
-				mem_map_lock_pages() ;
-				break;
 			default:
 				termch = cmd;
 				memex();
@@ -2458,249 +2442,6 @@ void mem_find_vsid()
 	printf("\nDone -------------------\n");
 }
 
-void mem_map_check_slab()
-{
-	int i, slab_count;
-
-	i = max_mapnr;
-	slab_count = 0;
-	
-	while (i-- > 0)  {
-		if (PageSlab(mem_map+i)){
-			printf(" slab entry - mem_map entry =%p  \n", mem_map+i);
-			slab_count ++;
-		}
-	}
-
-	printf(" count of pages for slab = %d \n", slab_count);
-}
-
-void mem_map_lock_pages()
-{
-	int i, lock_count;
-
-	i = max_mapnr;
-	lock_count = 0;
-	
-	while (i-- > 0)  {
-		if (PageLocked(mem_map+i)){
-			printf(" locked entry - mem_map entry =%p  \n", mem_map+i);
-			lock_count ++;
-		}
-	}
-
-	printf(" count of locked pages = %d \n", lock_count); 
-}
-
-void mem_check_dup_rpn ()
-{
-	unsigned long htab_size_bytes;
-	unsigned long htab_end;
-	unsigned long last_rpn;
-	HPTE *hpte1, *hpte2;
-	int dup_count;
-	struct task_struct *p;
-	unsigned long kernel_vsid_c0,kernel_vsid_c1,kernel_vsid_c2,kernel_vsid_c3;
-	unsigned long kernel_vsid_c4,kernel_vsid_c5,kernel_vsid_d,kernel_vsid_e;
-	unsigned long kernel_vsid_f;
-	unsigned long vsid0,vsid1,vsidB,vsid2;
-
-	htab_size_bytes = htab_data.htab_num_ptegs * 128; // 128B / PTEG
-	htab_end = (unsigned long)htab_data.htab + htab_size_bytes;
-	// last_rpn = (naca->physicalMemorySize-1) >> PAGE_SHIFT;
-	last_rpn = 0xfffff;
-
-	printf("\nHardware Page Table Check\n-------------------\n");
-	printf("htab base      : %.16lx\n", htab_data.htab);
-	printf("htab size      : %.16lx\n", htab_size_bytes);
-
-
-	for(hpte1 = htab_data.htab; hpte1 < (HPTE *)htab_end; hpte1++) {
-		if ( hpte1->dw0.dw0.v != 0 ) {
-			if ( hpte1->dw1.dw1.rpn <= last_rpn ) {
-				dup_count = 0;
-				for(hpte2 = hpte1+1; hpte2 < (HPTE *)htab_end; hpte2++) {
-					if ( hpte2->dw0.dw0.v != 0 ) {
-						if(hpte1->dw1.dw1.rpn == hpte2->dw1.dw1.rpn) {
-							dup_count++;
-						}
-					}
-				}
-				if(dup_count > 5) {
-					printf(" Duplicate rpn: %.13lx \n", (hpte1->dw1.dw1.rpn));
-					printf("    mem map array entry %p count = %d \n",
-					       (mem_map+(hpte1->dw1.dw1.rpn)), (mem_map+(hpte1->dw1.dw1.rpn))->count);
-					for(hpte2 = hpte1+1; hpte2 < (HPTE *)htab_end; hpte2++) {
-						if ( hpte2->dw0.dw0.v != 0 ) {
-							if(hpte1->dw1.dw1.rpn == hpte2->dw1.dw1.rpn) {
-								printf("   hpte2: %16.16lx  *hpte2: %16.16lx %16.16lx\n",
-								       hpte2, hpte2->dw0.dword0, hpte2->dw1.dword1);
-							}
-						}
-					}
-				}
-			} else {
-				printf(" Bogus rpn: %.13lx \n", (hpte1->dw1.dw1.rpn));
-				printf("   hpte: %16.16lx  *hpte: %16.16lx %16.16lx\n",
-				       hpte1, hpte1->dw0.dword0, hpte1->dw1.dword1);
-			}
-		}
-		if (xmon_interrupted())
-			return;
-	}                                                                     
-
-
-
-	// print the kernel vsids
-	kernel_vsid_c0 =  get_kernel_vsid(0xC000000000000000);
-	kernel_vsid_c1 =  get_kernel_vsid(0xC000000010000000);
-	kernel_vsid_c2 =  get_kernel_vsid(0xC000000020000000);
-	kernel_vsid_c3 =  get_kernel_vsid(0xC000000030000000);
-	kernel_vsid_c4 =  get_kernel_vsid(0xC000000040000000);
-	kernel_vsid_c5 =  get_kernel_vsid(0xC000000050000000);
-	kernel_vsid_d  =  get_kernel_vsid(0xD000000000000000);
-	kernel_vsid_e  =  get_kernel_vsid(0xE000000000000000);
-	kernel_vsid_f  =  get_kernel_vsid(0xF000000000000000);
-
-	printf(" kernel vsid -  seg c0  = %lx\n",  kernel_vsid_c0 );
-	printf(" kernel vsid -  seg c1  = %lx\n",  kernel_vsid_c1 );
-	printf(" kernel vsid -  seg c2  = %lx\n",  kernel_vsid_c2 );
-	printf(" kernel vsid -  seg c3  = %lx\n",  kernel_vsid_c3 );
-	printf(" kernel vsid -  seg c4  = %lx\n",  kernel_vsid_c4 );
-	printf(" kernel vsid -  seg c5  = %lx\n",  kernel_vsid_c5 );
-	printf(" kernel vsid -  seg d   = %lx\n",  kernel_vsid_d );
-	printf(" kernel vsid -  seg e   = %lx\n",  kernel_vsid_e );
-	printf(" kernel vsid -  seg f   = %lx\n",  kernel_vsid_f );
-
-
-  // print a list of valid vsids for the tasks
-	read_lock(&tasklist_lock);
-	for_each_task(p)
-		if(p->mm) {
-			struct mm_struct *mm = p->mm; 
-			printf(" task = %p  mm =  %lx  pgd   %lx\n", 
-			       p, mm, mm->pgd);
-			vsid0 = get_vsid( mm->context, 0 );
-			vsid1 = get_vsid( mm->context, 0x10000000 );
-			vsid2 = get_vsid( mm->context, 0x20000000 );
-			vsidB = get_vsid( mm->context, 0xB0000000 );
-			printf("            context = %lx  vsid seg 0  = %lx\n",  mm->context, vsid0 );
-			printf("                           vsid seg 1  = %lx\n",  vsid1 );
-			printf("                           vsid seg 2  = %lx\n",  vsid2 );
-			printf("                           vsid seg 2  = %lx\n",  vsidB );
-	  
-			printf("\n");
-		};
-	read_unlock(&tasklist_lock);
-
-	printf("\nDone -------------------\n");
-}
-
-
-
-void mem_check_pagetable_vsids ()
-{
-	unsigned long htab_size_bytes;
-	unsigned long htab_end;
-	unsigned long last_rpn;
-	struct task_struct *p;
-	unsigned long valid_table_count,invalid_table_count,bogus_rpn_count;
-	int found;
-	unsigned long user_address_table_count,kernel_page_table_count;
-	unsigned long pt_vsid;
-	HPTE *hpte1;
-
-
-	htab_size_bytes = htab_data.htab_num_ptegs * 128; // 128B / PTEG
-	htab_end = (unsigned long)htab_data.htab + htab_size_bytes;
-	// last_rpn = (naca->physicalMemorySize-1) >> PAGE_SHIFT;
-	last_rpn = 0xfffff;
-
-	printf("\nHardware Page Table Check\n-------------------\n");
-	printf("htab base      : %.16lx\n", htab_data.htab);
-	printf("htab size      : %.16lx\n", htab_size_bytes);
-
-	valid_table_count = 0;
-	invalid_table_count = 0;
-	bogus_rpn_count = 0;
-	user_address_table_count = 0;
-	kernel_page_table_count = 0;
-	for(hpte1 = htab_data.htab; hpte1 < (HPTE *)htab_end; hpte1++) {
-		if ( hpte1->dw0.dw0.v != 0 ) {
-			valid_table_count++;
-			if ( hpte1->dw1.dw1.rpn <= last_rpn ) {
-				pt_vsid =  (hpte1->dw0.dw0.avpn) >> 5;
-				if ((pt_vsid == get_kernel_vsid(0xC000000000000000)) |
-				    (pt_vsid == get_kernel_vsid(0xC000000010000000)) |
-				    (pt_vsid == get_kernel_vsid(0xC000000020000000)) |
-				    (pt_vsid == get_kernel_vsid(0xC000000030000000)) |
-				    (pt_vsid == get_kernel_vsid(0xC000000040000000)) |
-				    (pt_vsid == get_kernel_vsid(0xC000000050000000)) |
-				    (pt_vsid == get_kernel_vsid(0xD000000000000000)) |
-				    (pt_vsid == get_kernel_vsid(0xE000000000000000)) |
-				    (pt_vsid == get_kernel_vsid(0xF000000000000000)) ) {
-					kernel_page_table_count ++;
-				} else {
-					read_lock(&tasklist_lock);
-					found = 0;
-					for_each_task(p) {
-						if(p->mm && (found == 0)) {
-							struct mm_struct *mm = p->mm; 
-	     
-							if ((pt_vsid == get_vsid( mm->context, 0 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x10000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x20000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x30000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x40000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x50000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x60000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x70000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x80000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0x90000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0xA0000000 )) |
-							    (pt_vsid == get_vsid( mm->context, 0xB0000000 ))) {
-								user_address_table_count ++;
-								found = 1;
-							}
-						}
-					}
-					read_unlock(&tasklist_lock);
-					if (found == 0)
-					{
-						printf(" vsid not found vsid = %lx, hpte = %p  \n", 
-						       pt_vsid,hpte1);
-						printf("    rpn in entry = %lx \n", hpte1->dw1.dw1.rpn);
-						printf("    mem map address = %lx  \n", mem_map + (hpte1->dw1.dw1.rpn));
-           
-					} else //  found
-					{
-					}
-
-				}  // good rpn
-       
-			} else {
-				bogus_rpn_count ++;
-			}
-		} else {
-			invalid_table_count++;
-		}
-	}                                                                     
-
-
-	printf(" page table valid counts - valid entries = %lx  invalid entries =  %lx \n",
-	       valid_table_count, invalid_table_count);
-
-	printf("  bogus rpn entries ( probably io) = %lx \n", bogus_rpn_count);
-
- 
-  
-	printf(" page table counts - kernel entries = %lx  user entries =  %lx \n",
-	       kernel_page_table_count, user_address_table_count); 
-
-	printf("\nDone -------------------\n");
-
-}
-
 static void debug_trace(void) {
         unsigned long val, cmd, on;
 

include/asm-ppc64/bitops.h

@@ -259,6 +259,12 @@ static __inline__ int ffs(int x)
 	return __ilog2(i & -i) + 1;
 }
 
+/*
+ * fls: find last (most-significant) bit set.
+ * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
+ */
+#define fls(x) generic_fls(x)
+
 /*
  * hweightN: returns the hamming weight (i.e. the number
  * of bits set) of a N-bit word

include/asm-ppc64/mmzone.h

+/*
+ * Written by Kanoj Sarcar (kanoj@sgi.com) Aug 99
+ *
+ * PowerPC64 port:
+ * Copyright (C) 2002 Anton Blanchard, IBM Corp.
+ */
+#ifndef _ASM_MMZONE_H_
+#define _ASM_MMZONE_H_
+
+#include <linux/config.h>
+
+typedef struct plat_pglist_data {
+	pg_data_t	gendata;
+} plat_pg_data_t;
+
+/*
+ * Following are macros that are specific to this numa platform.
+ */
+
+extern plat_pg_data_t plat_node_data[];
+
+#define MAX_NUMNODES 4
+
+/* XXX grab this from the device tree - Anton */
+#define PHYSADDR_TO_NID(pa)		((pa) >> 36)
+#define PLAT_NODE_DATA(n)		(&plat_node_data[(n)])
+#define PLAT_NODE_DATA_STARTNR(n)	\
+	(PLAT_NODE_DATA(n)->gendata.node_start_mapnr)
+#define PLAT_NODE_DATA_SIZE(n)		(PLAT_NODE_DATA(n)->gendata.node_size)
+#define PLAT_NODE_DATA_LOCALNR(p, n)	\
+	(((p) - PLAT_NODE_DATA(n)->gendata.node_start_paddr) >> PAGE_SHIFT)
+
+#ifdef CONFIG_DISCONTIGMEM
+
+/*
+ * Following are macros that each numa implmentation must define.
+ */
+
+/*
+ * Given a kernel address, find the home node of the underlying memory.
+ */
+#define KVADDR_TO_NID(kaddr)	PHYSADDR_TO_NID(__pa(kaddr))
+
+/*
+ * Return a pointer to the node data for node n.
+ */
+#define NODE_DATA(n)	(&((PLAT_NODE_DATA(n))->gendata))
+
+/*
+ * NODE_MEM_MAP gives the kaddr for the mem_map of the node.
+ */
+#define NODE_MEM_MAP(nid)	(NODE_DATA(nid)->node_mem_map)
+
+/*
+ * Given a kaddr, ADDR_TO_MAPBASE finds the owning node of the memory
+ * and returns the mem_map of that node.
+ */
+#define ADDR_TO_MAPBASE(kaddr) \
+			NODE_MEM_MAP(KVADDR_TO_NID((unsigned long)(kaddr)))
+
+/*
+ * Given a kaddr, LOCAL_BASE_ADDR finds the owning node of the memory
+ * and returns the kaddr corresponding to first physical page in the
+ * node's mem_map.
+ */
+#define LOCAL_BASE_ADDR(kaddr) \
+	((unsigned long)__va(NODE_DATA(KVADDR_TO_NID(kaddr))->node_start_paddr))
+
+#define LOCAL_MAP_NR(kvaddr) \
+	(((unsigned long)(kvaddr)-LOCAL_BASE_ADDR(kvaddr)) >> PAGE_SHIFT)
+
+#if 0
+/* XXX fix - Anton */
+#define kern_addr_valid(kaddr)	test_bit(LOCAL_MAP_NR(kaddr), \
+					 NODE_DATA(KVADDR_TO_NID(kaddr))->valid_addr_bitmap)
+#endif
+
+#define discontigmem_pfn_to_page(pfn) \
+({ \
+	unsigned long kaddr = (unsigned long)__va(pfn << PAGE_SHIFT); \
+	(ADDR_TO_MAPBASE(kaddr) + LOCAL_MAP_NR(kaddr)); \
+})
+
+#ifdef CONFIG_NUMA
+
+/* XXX grab this from the device tree - Anton */
+#define cputonode(cpu)	((cpu) >> 3)
+
+#define numa_node_id()	cputonode(smp_processor_id())
+
+#endif /* CONFIG_NUMA */
+#endif /* CONFIG_DISCONTIGMEM */
+#endif /* _ASM_MMZONE_H_ */

include/asm-ppc64/page.h

@@ -70,8 +70,8 @@ static __inline__ void clear_page(void *addr)
 
 extern void copy_page(void *to, void *from);
 struct page;
-extern void clear_user_page(void *page, unsigned long vaddr);
-extern void copy_user_page(void *to, void *from, unsigned long vaddr);
+extern void clear_user_page(void *page, unsigned long vaddr, struct page *pg);
+extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *p);
 
 #ifdef STRICT_MM_TYPECHECKS
 /*
@@ -215,8 +215,15 @@ static inline int get_order(unsigned long size)
 #define __a2p(x) ((void *) absolute_to_phys(x))
 #define __a2v(x) ((void *) __va(absolute_to_phys(x)))
 
+#ifdef CONFIG_DISCONTIGMEM
+#define page_to_pfn(page) \
+		((page) - page_zone(page)->zone_mem_map + \
+		(page_zone(page)->zone_start_paddr >> PAGE_SHIFT))
+#define pfn_to_page(pfn)	discontigmem_pfn_to_page(pfn)
+#else
 #define pfn_to_page(pfn)	(mem_map + (pfn))
-#define page_to_pfn(pfn)	((unsigned long)((pfn) - mem_map))
+#define page_to_pfn(page)	((unsigned long)((page) - mem_map))
+#endif
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 
 #define pfn_valid(pfn)		((pfn) < max_mapnr)

include/asm-ppc64/pgalloc.h

@@ -53,6 +53,8 @@ pmd_free(pmd_t *pmd)
 	free_page((unsigned long)pmd);
 }
 
+#define pmd_free_tlb(tlb, pmd)	pmd_free(pmd)
+
 #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte)
 #define pmd_populate(mm, pmd, pte_page) \
 	pmd_populate_kernel(mm, pmd, page_address(pte_page))
@@ -86,6 +88,7 @@ pte_free_kernel(pte_t *pte)
 }
 
 #define pte_free(pte_page)	pte_free_kernel(page_address(pte_page))
+#define pte_free_tlb(tlb, pte)	pte_free(pte)
 
 #define check_pgt_cache()	do { } while (0)
 

include/asm-ppc64/tlb.h

+/*
+ *	TLB shootdown specifics for PPC64
+ *
+ * Copyright (C) 2002 Anton Blanchard, IBM Corp.
+ * Copyright (C) 2002 Paul Mackerras, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#ifndef _PPC64_TLB_H
+#define _PPC64_TLB_H
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+#include <asm/page.h>
+#include <asm/mmu.h>
+
+struct free_pte_ctx;
+static inline void tlb_flush(struct free_pte_ctx *tlb);
+
+/* Get the generic bits... */
 #include <asm-generic/tlb.h>
+
+/* Nothing needed here in fact... */
+#define tlb_start_vma(tlb, vma)	do { } while (0)
+#define tlb_end_vma(tlb, vma)	do { } while (0)
+
+/* Should make this at least as large as the generic batch size, but it
+ * takes up too much space */
+#define PPC64_TLB_BATCH_NR 192
+
+struct ppc64_tlb_batch {
+	unsigned long index;
+	pte_t pte[PPC64_TLB_BATCH_NR];
+	unsigned long addr[PPC64_TLB_BATCH_NR];
+	unsigned long vaddr[PPC64_TLB_BATCH_NR];
+};
+
+extern struct ppc64_tlb_batch ppc64_tlb_batch[NR_CPUS];
+
+static inline void tlb_remove_tlb_entry(mmu_gather_t *tlb, pte_t *ptep,
+					unsigned long address)
+{
+	int cpu = smp_processor_id();
+	struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[cpu];
+	unsigned long i = batch->index;
+	pte_t pte;
+
+	if (pte_val(*ptep) & _PAGE_HASHPTE) {
+		pte = __pte(pte_update(ptep, _PAGE_HPTEFLAGS, 0));
+		if (pte_val(pte) & _PAGE_HASHPTE) {
+			int local = 0;
+
+			if (tlb->mm->cpu_vm_mask == (1 << cpu))
+				local = 1;
+
+			batch->pte[i] = pte;
+			batch->addr[i] = address;
+			i++;
+
+			if (i == PPC64_TLB_BATCH_NR) {
+				flush_hash_range(tlb->mm->context, i, local);
+				i = 0;
+			}
+		}
+	}
+
+	batch->index = i;
+}
+
+static inline void tlb_flush(struct free_pte_ctx *tlb)
+{
+	int cpu = smp_processor_id();
+	struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[cpu];
+	int local = 0;
+
+	if (tlb->mm->cpu_vm_mask == (1 << smp_processor_id()))
+		local = 1;
+
+	flush_hash_range(tlb->mm->context, batch->index, local);
+	batch->index = 0;
+}
+
+#endif /* _PPC64_TLB_H */

include/asm-ppc64/tlbflush.h

@@ -35,12 +35,4 @@ extern void flush_hash_page(unsigned long context, unsigned long ea, pte_t pte,
 			    int local);
 void flush_hash_range(unsigned long context, unsigned long number, int local);
 
-/* TLB flush batching */
-#define MAX_BATCH_FLUSH 128
-struct tlb_batch_data {
-	pte_t pte;
-	unsigned long addr;
-};
-extern struct tlb_batch_data tlb_batch_array[NR_CPUS][MAX_BATCH_FLUSH];
-
 #endif /* _PPC64_TLBFLUSH_H */