Commit 6314b8e2 authored by Linus Torvalds's avatar Linus Torvalds

Split ptep_establish into "establish" and "update_access_flags"

ptep_establish() is used to establish a new mapping at COW time,
and it always replaces a non-writable page mapping with a totally
new page mapping that is dirty (and likely writable, although ptrace
may cause a non-writable new mapping). Because it was nonwritable,
we don't have to worry about losing concurrent dirty page bit updates.

ptep_update_access_flags() leaves the same page mapping, but updates
the accessed/dirty/writable bits (it only ever sets them, and never
removes any permissions). Often easier, but it may race with a dirty
bit update on another CPU.

Booted on x86 and ppc64.
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 5e681ee3
...@@ -2,22 +2,34 @@ ...@@ -2,22 +2,34 @@
#define _ASM_GENERIC_PGTABLE_H #define _ASM_GENERIC_PGTABLE_H
#ifndef __HAVE_ARCH_PTEP_ESTABLISH #ifndef __HAVE_ARCH_PTEP_ESTABLISH
#ifndef ptep_update_dirty_accessed
#define ptep_update_dirty_accessed(__ptep, __entry, __dirty) set_pte(__ptep, __entry)
#endif
/* /*
* Establish a new mapping: * Establish a new mapping:
* - flush the old one * - flush the old one
* - update the page tables * - update the page tables
* - inform the TLB about the new one * - inform the TLB about the new one
* *
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock.
*
* Note: the old pte is known to not be writable, so we don't need to
* worry about dirty bits etc getting lost.
*/
#define ptep_establish(__vma, __address, __ptep, __entry) \
do { \
set_pte(__ptep, __entry); \
flush_tlb_page(__vma, __address); \
} while (0)
#endif
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
* Largely same as above, but only sets the access flags (dirty,
* accessed, and writable). Furthermore, we know it always gets set
* to a "more permissive" setting, which allows most architectures
* to optimize this.
*/ */
#define ptep_establish(__vma, __address, __ptep, __entry, __dirty) \ #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
do { \ do { \
ptep_update_dirty_accessed(__ptep, __entry, __dirty); \ set_pte(__ptep, __entry); \
flush_tlb_page(__vma, __address); \ flush_tlb_page(__vma, __address); \
} while (0) } while (0)
#endif #endif
......
...@@ -325,9 +325,13 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) ...@@ -325,9 +325,13 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
* bit at the same time. * bit at the same time.
*/ */
#define update_mmu_cache(vma,address,pte) do { } while (0) #define update_mmu_cache(vma,address,pte) do { } while (0)
#define ptep_update_dirty_accessed(__ptep, __entry, __dirty) \ #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
do { \ do { \
if (__dirty) set_pte(__ptep, __entry); \ if (__dirty) { \
(__ptep)->pte_low = (__entry).pte_low; \
flush_tlb_page(__vma, __address); \
} \
} while (0) } while (0)
/* Encode and de-code a swap entry */ /* Encode and de-code a swap entry */
......
...@@ -548,6 +548,16 @@ static inline void ptep_mkdirty(pte_t *ptep) ...@@ -548,6 +548,16 @@ static inline void ptep_mkdirty(pte_t *ptep)
pte_update(ptep, 0, _PAGE_DIRTY); pte_update(ptep, 0, _PAGE_DIRTY);
} }
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
{
unsigned long bits = pte_val(entry) &
(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW);
pte_update(ptep, 0, bits);
}
#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
__ptep_set_access_flags(__ptep, __entry, __dirty)
/* /*
* Macro to mark a page protection value as "uncacheable". * Macro to mark a page protection value as "uncacheable".
*/ */
......
...@@ -306,7 +306,10 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr) ...@@ -306,7 +306,10 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr)
return old; return old;
} }
/* PTE updating functions */ /* PTE updating functions, this function puts the PTE in the
* batch, doesn't actually triggers the hash flush immediately,
* you need to call flush_tlb_pending() to do that.
*/
extern void hpte_update(pte_t *ptep, unsigned long pte, int wrprot); extern void hpte_update(pte_t *ptep, unsigned long pte, int wrprot);
static inline int ptep_test_and_clear_young(pte_t *ptep) static inline int ptep_test_and_clear_young(pte_t *ptep)
...@@ -318,7 +321,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep) ...@@ -318,7 +321,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep)
old = pte_update(ptep, _PAGE_ACCESSED); old = pte_update(ptep, _PAGE_ACCESSED);
if (old & _PAGE_HASHPTE) { if (old & _PAGE_HASHPTE) {
hpte_update(ptep, old, 0); hpte_update(ptep, old, 0);
flush_tlb_pending(); /* XXX generic code doesn't flush */ flush_tlb_pending();
} }
return (old & _PAGE_ACCESSED) != 0; return (old & _PAGE_ACCESSED) != 0;
} }
...@@ -396,11 +399,37 @@ static inline void pte_clear(pte_t * ptep) ...@@ -396,11 +399,37 @@ static inline void pte_clear(pte_t * ptep)
*/ */
static inline void set_pte(pte_t *ptep, pte_t pte) static inline void set_pte(pte_t *ptep, pte_t pte)
{ {
if (pte_present(*ptep)) if (pte_present(*ptep)) {
pte_clear(ptep); pte_clear(ptep);
flush_tlb_pending();
}
*ptep = __pte(pte_val(pte)) & ~_PAGE_HPTEFLAGS; *ptep = __pte(pte_val(pte)) & ~_PAGE_HPTEFLAGS;
} }
/* Set the dirty and/or accessed bits atomically in a linux PTE, this
* function doesn't need to flush the hash entry
*/
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
{
unsigned long bits = pte_val(entry) &
(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW);
unsigned long old, tmp;
__asm__ __volatile__(
"1: ldarx %0,0,%4\n\
andi. %1,%0,%6\n\
bne- 1b \n\
or %0,%3,%0\n\
stdcx. %0,0,%4\n\
bne- 1b"
:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
:"r" (bits), "r" (ptep), "m" (ptep), "i" (_PAGE_BUSY)
:"cc");
}
#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
__ptep_set_access_flags(__ptep, __entry, __dirty)
/* /*
* Macro to mark a page protection value as "uncacheable". * Macro to mark a page protection value as "uncacheable".
*/ */
......
...@@ -581,7 +581,7 @@ static inline void ptep_mkdirty(pte_t *ptep) ...@@ -581,7 +581,7 @@ static inline void ptep_mkdirty(pte_t *ptep)
static inline void static inline void
ptep_establish(struct vm_area_struct *vma, ptep_establish(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep, unsigned long address, pte_t *ptep,
pte_t entry, int dirty) pte_t entry)
{ {
ptep_clear_flush(vma, address, ptep); ptep_clear_flush(vma, address, ptep);
set_pte(ptep, entry); set_pte(ptep, entry);
......
...@@ -1004,7 +1004,7 @@ static inline void break_cow(struct vm_area_struct * vma, struct page * new_page ...@@ -1004,7 +1004,7 @@ static inline void break_cow(struct vm_area_struct * vma, struct page * new_page
flush_cache_page(vma, address); flush_cache_page(vma, address);
entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)), entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)),
vma); vma);
ptep_establish(vma, address, page_table, entry, 1); ptep_establish(vma, address, page_table, entry);
update_mmu_cache(vma, address, entry); update_mmu_cache(vma, address, entry);
} }
...@@ -1056,7 +1056,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma, ...@@ -1056,7 +1056,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
flush_cache_page(vma, address); flush_cache_page(vma, address);
entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)), entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)),
vma); vma);
ptep_establish(vma, address, page_table, entry, 1); ptep_set_access_flags(vma, address, page_table, entry, 1);
update_mmu_cache(vma, address, entry); update_mmu_cache(vma, address, entry);
pte_unmap(page_table); pte_unmap(page_table);
spin_unlock(&mm->page_table_lock); spin_unlock(&mm->page_table_lock);
...@@ -1646,7 +1646,7 @@ static inline int handle_pte_fault(struct mm_struct *mm, ...@@ -1646,7 +1646,7 @@ static inline int handle_pte_fault(struct mm_struct *mm,
entry = pte_mkdirty(entry); entry = pte_mkdirty(entry);
} }
entry = pte_mkyoung(entry); entry = pte_mkyoung(entry);
ptep_establish(vma, address, pte, entry, write_access); ptep_set_access_flags(vma, address, pte, entry, write_access);
update_mmu_cache(vma, address, entry); update_mmu_cache(vma, address, entry);
pte_unmap(pte); pte_unmap(pte);
spin_unlock(&mm->page_table_lock); spin_unlock(&mm->page_table_lock);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment