Commit 33d55a8a authored by Jack Steiner's avatar Jack Steiner Committed by David Mosberger

[PATCH] ia64: Enable cpu_vm_mask maintenance and improve SN2 TLB flushing

parent 7b080b4c
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
* License. See the file "COPYING" in the main directory of this archive * License. See the file "COPYING" in the main directory of this archive
* for more details. * for more details.
* *
* Copyright (C) 2000-2003 Silicon Graphics, Inc. All rights reserved. * Copyright (C) 2000-2004 Silicon Graphics, Inc. All rights reserved.
*/ */
#include <linux/init.h> #include <linux/init.h>
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
#include <asm/delay.h> #include <asm/delay.h>
#include <asm/io.h> #include <asm/io.h>
#include <asm/smp.h> #include <asm/smp.h>
#include <asm/numa.h>
#include <asm/bitops.h>
#include <asm/hw_irq.h> #include <asm/hw_irq.h>
#include <asm/current.h> #include <asm/current.h>
#include <asm/sn/sn_cpuid.h> #include <asm/sn/sn_cpuid.h>
...@@ -67,14 +69,56 @@ wait_piowc(void) ...@@ -67,14 +69,56 @@ wait_piowc(void)
* *
* Purges the translation caches of all processors of the given virtual address * Purges the translation caches of all processors of the given virtual address
* range. * range.
*
* Note:
* - cpu_vm_mask is a bit mask that indicates which cpus have loaded the context.
* - cpu_vm_mask is converted into a nodemask of the nodes containing the
* cpus in cpu_vm_mask.
* - if only one bit is set in cpu_vm_mask & it is the current cpu,
* then only the local TLB needs to be flushed. This flushing can be done
* using ptc.l. This is the common case & avoids the global spinlock.
* - if multiple cpus have loaded the context, then flushing has to be
* done with ptc.g/MMRs under protection of the global ptc_lock.
*/ */
void void
sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits) sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits)
{ {
int cnode, mycnode, nasid, flushed=0; int i, cnode, mynasid, cpu, lcpu=0, nasid, flushed=0;
volatile unsigned long *ptc0, *ptc1; volatile unsigned long *ptc0, *ptc1;
unsigned long flags=0, data0, data1; unsigned long flags=0, data0, data1;
struct mm_struct *mm=current->active_mm;
short nasids[NR_NODES], nix;
DECLARE_BITMAP(nodes_flushed, NR_NODES);
CLEAR_BITMAP(nodes_flushed, NR_NODES);
i = 0;
for_each_cpu_mask(cpu, mm->cpu_vm_mask) {
cnode = cpu_to_node(cpu);
__set_bit(cnode, nodes_flushed);
lcpu = cpu;
i++;
}
preempt_disable();
if (likely(i == 1 && lcpu == smp_processor_id())) {
do {
ia64_ptcl(start, nbits<<2);
start += (1UL << nbits);
} while (start < end);
ia64_srlz_i();
preempt_enable();
return;
}
nix = 0;
for (cnode=find_first_bit(&nodes_flushed, NR_NODES); cnode < NR_NODES;
cnode=find_next_bit(&nodes_flushed, NR_NODES, ++cnode))
nasids[nix++] = cnodeid_to_nasid(cnode);
data0 = (1UL<<SH_PTC_0_A_SHFT) | data0 = (1UL<<SH_PTC_0_A_SHFT) |
(nbits<<SH_PTC_0_PS_SHFT) | (nbits<<SH_PTC_0_PS_SHFT) |
...@@ -84,20 +128,19 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit ...@@ -84,20 +128,19 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit
ptc0 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_0); ptc0 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_0);
ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1); ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1);
mycnode = numa_node_id();
mynasid = smp_physical_node_id();
spin_lock_irqsave(&sn2_global_ptc_lock, flags); spin_lock_irqsave(&sn2_global_ptc_lock, flags);
do { do {
data1 = start | (1UL<<SH_PTC_1_START_SHFT); data1 = start | (1UL<<SH_PTC_1_START_SHFT);
for (cnode = 0; cnode < numnodes; cnode++) { for (i=0; i<nix; i++) {
if (is_headless_node(cnode)) nasid = nasids[i];
continue; if (likely(nasid == mynasid)) {
if (cnode == mycnode) {
ia64_ptcga(start, nbits<<2); ia64_ptcga(start, nbits<<2);
ia64_srlz_i(); ia64_srlz_i();
} else { } else {
nasid = cnodeid_to_nasid(cnode);
ptc0 = CHANGE_NASID(nasid, ptc0); ptc0 = CHANGE_NASID(nasid, ptc0);
ptc1 = CHANGE_NASID(nasid, ptc1); ptc1 = CHANGE_NASID(nasid, ptc1);
pio_atomic_phys_write_mmrs(ptc0, data0, ptc1, data1); pio_atomic_phys_write_mmrs(ptc0, data0, ptc1, data1);
...@@ -115,6 +158,7 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit ...@@ -115,6 +158,7 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit
spin_unlock_irqrestore(&sn2_global_ptc_lock, flags); spin_unlock_irqrestore(&sn2_global_ptc_lock, flags);
preempt_enable();
} }
/* /*
......
...@@ -106,6 +106,7 @@ get_mmu_context (struct mm_struct *mm) ...@@ -106,6 +106,7 @@ get_mmu_context (struct mm_struct *mm)
/* re-check, now that we've got the lock: */ /* re-check, now that we've got the lock: */
context = mm->context; context = mm->context;
if (context == 0) { if (context == 0) {
cpus_clear(mm->cpu_vm_mask);
if (ia64_ctx.next >= ia64_ctx.limit) if (ia64_ctx.next >= ia64_ctx.limit)
wrap_mmu_context(mm); wrap_mmu_context(mm);
mm->context = context = ia64_ctx.next++; mm->context = context = ia64_ctx.next++;
...@@ -170,6 +171,8 @@ activate_context (struct mm_struct *mm) ...@@ -170,6 +171,8 @@ activate_context (struct mm_struct *mm)
do { do {
context = get_mmu_context(mm); context = get_mmu_context(mm);
MMU_TRACE('A', smp_processor_id(), mm, context); MMU_TRACE('A', smp_processor_id(), mm, context);
if (!cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
cpu_set(smp_processor_id(), mm->cpu_vm_mask);
reload_context(context); reload_context(context);
MMU_TRACE('a', smp_processor_id(), mm, context); MMU_TRACE('a', smp_processor_id(), mm, context);
/* in the unlikely event of a TLB-flush by another thread, redo the load: */ /* in the unlikely event of a TLB-flush by another thread, redo the load: */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment