Commit 637ff9ef authored by Janosch Frank's avatar Janosch Frank

s390/mm: Add huge pmd storage key handling

Storage keys for guests with huge page mappings have to be managed in
hardware. There are no PGSTEs for PMDs that we could use to retain the
guests's logical view of the key.
Signed-off-by: default avatarJanosch Frank <frankja@linux.vnet.ibm.com>
Reviewed-by: default avatarDavid Hildenbrand <david@redhat.com>
parent 3afdfca6
...@@ -410,6 +410,24 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, ...@@ -410,6 +410,24 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
return old; return old;
} }
static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
p4d = p4d_alloc(mm, pgd, addr);
if (!p4d)
return NULL;
pud = pud_alloc(mm, p4d, addr);
if (!pud)
return NULL;
pmd = pmd_alloc(mm, pud, addr);
return pmd;
}
pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t new) pmd_t *pmdp, pmd_t new)
{ {
...@@ -734,12 +752,36 @@ EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc); ...@@ -734,12 +752,36 @@ EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
unsigned char key, bool nq) unsigned char key, bool nq)
{ {
unsigned long keyul; unsigned long keyul, paddr;
spinlock_t *ptl; spinlock_t *ptl;
pgste_t old, new; pgste_t old, new;
pmd_t *pmdp;
pte_t *ptep; pte_t *ptep;
ptep = get_locked_pte(mm, addr, &ptl); pmdp = pmd_alloc_map(mm, addr);
if (unlikely(!pmdp))
return -EFAULT;
ptl = pmd_lock(mm, pmdp);
if (!pmd_present(*pmdp)) {
spin_unlock(ptl);
return -EFAULT;
}
if (pmd_large(*pmdp)) {
paddr = pmd_val(*pmdp) & HPAGE_MASK;
paddr |= addr & ~HPAGE_MASK;
/*
* Huge pmds need quiescing operations, they are
* always mapped.
*/
page_set_storage_key(paddr, key, 1);
spin_unlock(ptl);
return 0;
}
spin_unlock(ptl);
ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
if (unlikely(!ptep)) if (unlikely(!ptep))
return -EFAULT; return -EFAULT;
...@@ -750,14 +792,14 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, ...@@ -750,14 +792,14 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
if (!(pte_val(*ptep) & _PAGE_INVALID)) { if (!(pte_val(*ptep) & _PAGE_INVALID)) {
unsigned long address, bits, skey; unsigned long bits, skey;
address = pte_val(*ptep) & PAGE_MASK; paddr = pte_val(*ptep) & PAGE_MASK;
skey = (unsigned long) page_get_storage_key(address); skey = (unsigned long) page_get_storage_key(paddr);
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
/* Set storage key ACC and FP */ /* Set storage key ACC and FP */
page_set_storage_key(address, skey, !nq); page_set_storage_key(paddr, skey, !nq);
/* Merge host changed & referenced into pgste */ /* Merge host changed & referenced into pgste */
pgste_val(new) |= bits << 52; pgste_val(new) |= bits << 52;
} }
...@@ -813,11 +855,32 @@ EXPORT_SYMBOL(cond_set_guest_storage_key); ...@@ -813,11 +855,32 @@ EXPORT_SYMBOL(cond_set_guest_storage_key);
int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
{ {
spinlock_t *ptl; spinlock_t *ptl;
unsigned long paddr;
pgste_t old, new; pgste_t old, new;
pmd_t *pmdp;
pte_t *ptep; pte_t *ptep;
int cc = 0; int cc = 0;
ptep = get_locked_pte(mm, addr, &ptl); pmdp = pmd_alloc_map(mm, addr);
if (unlikely(!pmdp))
return -EFAULT;
ptl = pmd_lock(mm, pmdp);
if (!pmd_present(*pmdp)) {
spin_unlock(ptl);
return -EFAULT;
}
if (pmd_large(*pmdp)) {
paddr = pmd_val(*pmdp) & HPAGE_MASK;
paddr |= addr & ~HPAGE_MASK;
cc = page_reset_referenced(paddr);
spin_unlock(ptl);
return cc;
}
spin_unlock(ptl);
ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
if (unlikely(!ptep)) if (unlikely(!ptep))
return -EFAULT; return -EFAULT;
...@@ -826,7 +889,8 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) ...@@ -826,7 +889,8 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
pgste_val(new) &= ~PGSTE_GR_BIT; pgste_val(new) &= ~PGSTE_GR_BIT;
if (!(pte_val(*ptep) & _PAGE_INVALID)) { if (!(pte_val(*ptep) & _PAGE_INVALID)) {
cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); paddr = pte_val(*ptep) & PAGE_MASK;
cc = page_reset_referenced(paddr);
/* Merge real referenced bit into host-set */ /* Merge real referenced bit into host-set */
pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
} }
...@@ -845,18 +909,42 @@ EXPORT_SYMBOL(reset_guest_reference_bit); ...@@ -845,18 +909,42 @@ EXPORT_SYMBOL(reset_guest_reference_bit);
int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
unsigned char *key) unsigned char *key)
{ {
unsigned long paddr;
spinlock_t *ptl; spinlock_t *ptl;
pgste_t pgste; pgste_t pgste;
pmd_t *pmdp;
pte_t *ptep; pte_t *ptep;
ptep = get_locked_pte(mm, addr, &ptl); pmdp = pmd_alloc_map(mm, addr);
if (unlikely(!pmdp))
return -EFAULT;
ptl = pmd_lock(mm, pmdp);
if (!pmd_present(*pmdp)) {
/* Not yet mapped memory has a zero key */
spin_unlock(ptl);
*key = 0;
return 0;
}
if (pmd_large(*pmdp)) {
paddr = pmd_val(*pmdp) & HPAGE_MASK;
paddr |= addr & ~HPAGE_MASK;
*key = page_get_storage_key(paddr);
spin_unlock(ptl);
return 0;
}
spin_unlock(ptl);
ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
if (unlikely(!ptep)) if (unlikely(!ptep))
return -EFAULT; return -EFAULT;
pgste = pgste_get_lock(ptep); pgste = pgste_get_lock(ptep);
*key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
paddr = pte_val(*ptep) & PAGE_MASK;
if (!(pte_val(*ptep) & _PAGE_INVALID)) if (!(pte_val(*ptep) & _PAGE_INVALID))
*key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); *key = page_get_storage_key(paddr);
/* Reflect guest's logical view, not physical */ /* Reflect guest's logical view, not physical */
*key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
pgste_set_unlock(ptep, pgste); pgste_set_unlock(ptep, pgste);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment