Commit c389a250 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

mm, thp: do not access mm->pmd_huge_pte directly

Currently mm->pmd_huge_pte protected by page table lock.  It will not
work with split lock.  We have to have per-pmd pmd_huge_pte for proper
access serialization.

For now, let's just introduce wrapper to access mm->pmd_huge_pte.
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: default avatarAlex Thorlton <athorlton@sgi.com>
Cc: Alex Thorlton <athorlton@sgi.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: "Eric W . Biederman" <ebiederm@xmission.com>
Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Jones <davej@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Robin Holt <robinmholt@gmail.com>
Cc: Sedat Dilek <sedat.dilek@gmail.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 117b0791
...@@ -1244,11 +1244,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, ...@@ -1244,11 +1244,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
if (!mm->pmd_huge_pte) if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(lh); INIT_LIST_HEAD(lh);
else else
list_add(lh, (struct list_head *) mm->pmd_huge_pte); list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
mm->pmd_huge_pte = pgtable; pmd_huge_pte(mm, pmdp) = pgtable;
} }
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
...@@ -1260,12 +1260,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) ...@@ -1260,12 +1260,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
pgtable = mm->pmd_huge_pte; pgtable = pmd_huge_pte(mm, pmdp);
lh = (struct list_head *) pgtable; lh = (struct list_head *) pgtable;
if (list_empty(lh)) if (list_empty(lh))
mm->pmd_huge_pte = NULL; pmd_huge_pte(mm, pmdp) = NULL;
else { else {
mm->pmd_huge_pte = (pgtable_t) lh->next; pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
list_del(lh); list_del(lh);
} }
ptep = (pte_t *) pgtable; ptep = (pte_t *) pgtable;
......
...@@ -196,11 +196,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, ...@@ -196,11 +196,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
if (!mm->pmd_huge_pte) if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(lh); INIT_LIST_HEAD(lh);
else else
list_add(lh, (struct list_head *) mm->pmd_huge_pte); list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
mm->pmd_huge_pte = pgtable; pmd_huge_pte(mm, pmdp) = pgtable;
} }
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
...@@ -211,12 +211,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) ...@@ -211,12 +211,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
pgtable = mm->pmd_huge_pte; pgtable = pmd_huge_pte(mm, pmdp);
lh = (struct list_head *) pgtable; lh = (struct list_head *) pgtable;
if (list_empty(lh)) if (list_empty(lh))
mm->pmd_huge_pte = NULL; pmd_huge_pte(mm, pmdp) = NULL;
else { else {
mm->pmd_huge_pte = (pgtable_t) lh->next; pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
list_del(lh); list_del(lh);
} }
pte_val(pgtable[0]) = 0; pte_val(pgtable[0]) = 0;
......
...@@ -1383,6 +1383,7 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) ...@@ -1383,6 +1383,7 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
return &mm->page_table_lock; return &mm->page_table_lock;
} }
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
{ {
......
...@@ -154,11 +154,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, ...@@ -154,11 +154,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
if (!mm->pmd_huge_pte) if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(&pgtable->lru); INIT_LIST_HEAD(&pgtable->lru);
else else
list_add(&pgtable->lru, &mm->pmd_huge_pte->lru); list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
mm->pmd_huge_pte = pgtable; pmd_huge_pte(mm, pmdp) = pgtable;
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif #endif
...@@ -173,11 +173,11 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) ...@@ -173,11 +173,11 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
assert_spin_locked(&mm->page_table_lock); assert_spin_locked(&mm->page_table_lock);
/* FIFO */ /* FIFO */
pgtable = mm->pmd_huge_pte; pgtable = pmd_huge_pte(mm, pmdp);
if (list_empty(&pgtable->lru)) if (list_empty(&pgtable->lru))
mm->pmd_huge_pte = NULL; pmd_huge_pte(mm, pmdp) = NULL;
else { else {
mm->pmd_huge_pte = list_entry(pgtable->lru.next, pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
struct page, lru); struct page, lru);
list_del(&pgtable->lru); list_del(&pgtable->lru);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment