Commit f3207c12 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman

powerpc/mm/slice: Convert slice_mask high slice to a bitmap

In followup patch we want to increase the va range which will result
in us requiring high_slices to have more than 64 bits. To enable this
convert high_slices to bitmap. We keep the number bits same in this patch
and later change that to higher value
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Fold in fix to use bitmap_empty()]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 6aa59f51
...@@ -98,19 +98,16 @@ extern u64 ppc64_pft_size; ...@@ -98,19 +98,16 @@ extern u64 ppc64_pft_size;
#define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT) #define GET_LOW_SLICE_INDEX(addr) ((addr) >> SLICE_LOW_SHIFT)
#define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT) #define GET_HIGH_SLICE_INDEX(addr) ((addr) >> SLICE_HIGH_SHIFT)
#ifndef __ASSEMBLY__
/* /*
* 1 bit per slice and we have one slice per 1TB * One bit per slice. We have lower slices which cover 256MB segments
* Right now we support only 64TB. * upto 4G range. That gets us 16 low slices. For the rest we track slices
* IF we change this we will have to change the type * in 1TB size.
* of high_slices * 64 below is actually SLICE_NUM_HIGH to fixup complie errros
*/ */
#define SLICE_MASK_SIZE 8
#ifndef __ASSEMBLY__
struct slice_mask { struct slice_mask {
u16 low_slices; u16 low_slices;
u64 high_slices; DECLARE_BITMAP(high_slices, 64);
}; };
struct mm_struct; struct mm_struct;
......
...@@ -36,11 +36,6 @@ ...@@ -36,11 +36,6 @@
#include <asm/copro.h> #include <asm/copro.h>
#include <asm/hugetlb.h> #include <asm/hugetlb.h>
/* some sanity checks */
#if (H_PGTABLE_RANGE >> 43) > SLICE_MASK_SIZE
#error H_PGTABLE_RANGE exceeds slice_mask high_slices size
#endif
static DEFINE_SPINLOCK(slice_convert_lock); static DEFINE_SPINLOCK(slice_convert_lock);
...@@ -49,7 +44,7 @@ int _slice_debug = 1; ...@@ -49,7 +44,7 @@ int _slice_debug = 1;
static void slice_print_mask(const char *label, struct slice_mask mask) static void slice_print_mask(const char *label, struct slice_mask mask)
{ {
char *p, buf[16 + 3 + 64 + 1]; char *p, buf[SLICE_NUM_LOW + 3 + SLICE_NUM_HIGH + 1];
int i; int i;
if (!_slice_debug) if (!_slice_debug)
...@@ -60,8 +55,12 @@ static void slice_print_mask(const char *label, struct slice_mask mask) ...@@ -60,8 +55,12 @@ static void slice_print_mask(const char *label, struct slice_mask mask)
*(p++) = ' '; *(p++) = ' ';
*(p++) = '-'; *(p++) = '-';
*(p++) = ' '; *(p++) = ' ';
for (i = 0; i < SLICE_NUM_HIGH; i++) for (i = 0; i < SLICE_NUM_HIGH; i++) {
*(p++) = (mask.high_slices & (1ul << i)) ? '1' : '0'; if (test_bit(i, mask.high_slices))
*(p++) = '1';
else
*(p++) = '0';
}
*(p++) = 0; *(p++) = 0;
printk(KERN_DEBUG "%s:%s\n", label, buf); printk(KERN_DEBUG "%s:%s\n", label, buf);
...@@ -80,7 +79,10 @@ static struct slice_mask slice_range_to_mask(unsigned long start, ...@@ -80,7 +79,10 @@ static struct slice_mask slice_range_to_mask(unsigned long start,
unsigned long len) unsigned long len)
{ {
unsigned long end = start + len - 1; unsigned long end = start + len - 1;
struct slice_mask ret = { 0, 0 }; struct slice_mask ret;
ret.low_slices = 0;
bitmap_zero(ret.high_slices, SLICE_NUM_HIGH);
if (start < SLICE_LOW_TOP) { if (start < SLICE_LOW_TOP) {
unsigned long mend = min(end, (SLICE_LOW_TOP - 1)); unsigned long mend = min(end, (SLICE_LOW_TOP - 1));
...@@ -89,10 +91,13 @@ static struct slice_mask slice_range_to_mask(unsigned long start, ...@@ -89,10 +91,13 @@ static struct slice_mask slice_range_to_mask(unsigned long start,
- (1u << GET_LOW_SLICE_INDEX(start)); - (1u << GET_LOW_SLICE_INDEX(start));
} }
if ((start + len) > SLICE_LOW_TOP) if ((start + len) > SLICE_LOW_TOP) {
ret.high_slices = (1ul << (GET_HIGH_SLICE_INDEX(end) + 1)) unsigned long start_index = GET_HIGH_SLICE_INDEX(start);
- (1ul << GET_HIGH_SLICE_INDEX(start)); unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT));
unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index;
bitmap_set(ret.high_slices, start_index, count);
}
return ret; return ret;
} }
...@@ -129,9 +134,12 @@ static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice) ...@@ -129,9 +134,12 @@ static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
static struct slice_mask slice_mask_for_free(struct mm_struct *mm) static struct slice_mask slice_mask_for_free(struct mm_struct *mm)
{ {
struct slice_mask ret = { 0, 0 }; struct slice_mask ret;
unsigned long i; unsigned long i;
ret.low_slices = 0;
bitmap_zero(ret.high_slices, SLICE_NUM_HIGH);
for (i = 0; i < SLICE_NUM_LOW; i++) for (i = 0; i < SLICE_NUM_LOW; i++)
if (!slice_low_has_vma(mm, i)) if (!slice_low_has_vma(mm, i))
ret.low_slices |= 1u << i; ret.low_slices |= 1u << i;
...@@ -141,7 +149,7 @@ static struct slice_mask slice_mask_for_free(struct mm_struct *mm) ...@@ -141,7 +149,7 @@ static struct slice_mask slice_mask_for_free(struct mm_struct *mm)
for (i = 0; i < SLICE_NUM_HIGH; i++) for (i = 0; i < SLICE_NUM_HIGH; i++)
if (!slice_high_has_vma(mm, i)) if (!slice_high_has_vma(mm, i))
ret.high_slices |= 1ul << i; __set_bit(i, ret.high_slices);
return ret; return ret;
} }
...@@ -150,10 +158,13 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize) ...@@ -150,10 +158,13 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize)
{ {
unsigned char *hpsizes; unsigned char *hpsizes;
int index, mask_index; int index, mask_index;
struct slice_mask ret = { 0, 0 }; struct slice_mask ret;
unsigned long i; unsigned long i;
u64 lpsizes; u64 lpsizes;
ret.low_slices = 0;
bitmap_zero(ret.high_slices, SLICE_NUM_HIGH);
lpsizes = mm->context.low_slices_psize; lpsizes = mm->context.low_slices_psize;
for (i = 0; i < SLICE_NUM_LOW; i++) for (i = 0; i < SLICE_NUM_LOW; i++)
if (((lpsizes >> (i * 4)) & 0xf) == psize) if (((lpsizes >> (i * 4)) & 0xf) == psize)
...@@ -164,7 +175,7 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize) ...@@ -164,7 +175,7 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize)
mask_index = i & 0x1; mask_index = i & 0x1;
index = i >> 1; index = i >> 1;
if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == psize) if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == psize)
ret.high_slices |= 1ul << i; __set_bit(i, ret.high_slices);
} }
return ret; return ret;
...@@ -172,8 +183,13 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize) ...@@ -172,8 +183,13 @@ static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize)
static int slice_check_fit(struct slice_mask mask, struct slice_mask available) static int slice_check_fit(struct slice_mask mask, struct slice_mask available)
{ {
DECLARE_BITMAP(result, SLICE_NUM_HIGH);
bitmap_and(result, mask.high_slices,
available.high_slices, SLICE_NUM_HIGH);
return (mask.low_slices & available.low_slices) == mask.low_slices && return (mask.low_slices & available.low_slices) == mask.low_slices &&
(mask.high_slices & available.high_slices) == mask.high_slices; bitmap_equal(result, mask.high_slices, SLICE_NUM_HIGH);
} }
static void slice_flush_segments(void *parm) static void slice_flush_segments(void *parm)
...@@ -220,7 +236,7 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz ...@@ -220,7 +236,7 @@ static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psiz
for (i = 0; i < SLICE_NUM_HIGH; i++) { for (i = 0; i < SLICE_NUM_HIGH; i++) {
mask_index = i & 0x1; mask_index = i & 0x1;
index = i >> 1; index = i >> 1;
if (mask.high_slices & (1ul << i)) if (test_bit(i, mask.high_slices))
hpsizes[index] = (hpsizes[index] & hpsizes[index] = (hpsizes[index] &
~(0xf << (mask_index * 4))) | ~(0xf << (mask_index * 4))) |
(((unsigned long)psize) << (mask_index * 4)); (((unsigned long)psize) << (mask_index * 4));
...@@ -256,7 +272,7 @@ static bool slice_scan_available(unsigned long addr, ...@@ -256,7 +272,7 @@ static bool slice_scan_available(unsigned long addr,
slice = GET_HIGH_SLICE_INDEX(addr); slice = GET_HIGH_SLICE_INDEX(addr);
*boundary_addr = (slice + end) ? *boundary_addr = (slice + end) ?
((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP; ((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP;
return !!(available.high_slices & (1ul << slice)); return !!test_bit(slice, available.high_slices);
} }
} }
...@@ -363,15 +379,24 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len, ...@@ -363,15 +379,24 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
return slice_find_area_bottomup(mm, len, mask, psize); return slice_find_area_bottomup(mm, len, mask, psize);
} }
#define or_mask(dst, src) do { \ static inline void slice_or_mask(struct slice_mask *dst, struct slice_mask *src)
(dst).low_slices |= (src).low_slices; \ {
(dst).high_slices |= (src).high_slices; \ DECLARE_BITMAP(result, SLICE_NUM_HIGH);
} while (0)
#define andnot_mask(dst, src) do { \ dst->low_slices |= src->low_slices;
(dst).low_slices &= ~(src).low_slices; \ bitmap_or(result, dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
(dst).high_slices &= ~(src).high_slices; \ bitmap_copy(dst->high_slices, result, SLICE_NUM_HIGH);
} while (0) }
static inline void slice_andnot_mask(struct slice_mask *dst, struct slice_mask *src)
{
DECLARE_BITMAP(result, SLICE_NUM_HIGH);
dst->low_slices &= ~src->low_slices;
bitmap_andnot(result, dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
bitmap_copy(dst->high_slices, result, SLICE_NUM_HIGH);
}
#ifdef CONFIG_PPC_64K_PAGES #ifdef CONFIG_PPC_64K_PAGES
#define MMU_PAGE_BASE MMU_PAGE_64K #define MMU_PAGE_BASE MMU_PAGE_64K
...@@ -383,15 +408,28 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -383,15 +408,28 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
unsigned long flags, unsigned int psize, unsigned long flags, unsigned int psize,
int topdown) int topdown)
{ {
struct slice_mask mask = {0, 0}; struct slice_mask mask;
struct slice_mask good_mask; struct slice_mask good_mask;
struct slice_mask potential_mask = {0,0} /* silence stupid warning */; struct slice_mask potential_mask;
struct slice_mask compat_mask = {0, 0}; struct slice_mask compat_mask;
int fixed = (flags & MAP_FIXED); int fixed = (flags & MAP_FIXED);
int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT); int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
unsigned long newaddr; unsigned long newaddr;
/*
* init different masks
*/
mask.low_slices = 0;
bitmap_zero(mask.high_slices, SLICE_NUM_HIGH);
/* silence stupid warning */;
potential_mask.low_slices = 0;
bitmap_zero(potential_mask.high_slices, SLICE_NUM_HIGH);
compat_mask.low_slices = 0;
bitmap_zero(compat_mask.high_slices, SLICE_NUM_HIGH);
/* Sanity checks */ /* Sanity checks */
BUG_ON(mm->task_size == 0); BUG_ON(mm->task_size == 0);
VM_BUG_ON(radix_enabled()); VM_BUG_ON(radix_enabled());
...@@ -449,7 +487,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -449,7 +487,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
if (psize == MMU_PAGE_64K) { if (psize == MMU_PAGE_64K) {
compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K); compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
if (fixed) if (fixed)
or_mask(good_mask, compat_mask); slice_or_mask(&good_mask, &compat_mask);
} }
#endif #endif
...@@ -484,7 +522,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -484,7 +522,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
* empty and thus can be converted * empty and thus can be converted
*/ */
potential_mask = slice_mask_for_free(mm); potential_mask = slice_mask_for_free(mm);
or_mask(potential_mask, good_mask); slice_or_mask(&potential_mask, &good_mask);
slice_print_mask(" potential", potential_mask); slice_print_mask(" potential", potential_mask);
if ((addr != 0 || fixed) && slice_check_fit(mask, potential_mask)) { if ((addr != 0 || fixed) && slice_check_fit(mask, potential_mask)) {
...@@ -517,7 +555,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -517,7 +555,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
#ifdef CONFIG_PPC_64K_PAGES #ifdef CONFIG_PPC_64K_PAGES
if (addr == -ENOMEM && psize == MMU_PAGE_64K) { if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
/* retry the search with 4k-page slices included */ /* retry the search with 4k-page slices included */
or_mask(potential_mask, compat_mask); slice_or_mask(&potential_mask, &compat_mask);
addr = slice_find_area(mm, len, potential_mask, psize, addr = slice_find_area(mm, len, potential_mask, psize,
topdown); topdown);
} }
...@@ -531,9 +569,9 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -531,9 +569,9 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
slice_print_mask(" mask", mask); slice_print_mask(" mask", mask);
convert: convert:
andnot_mask(mask, good_mask); slice_andnot_mask(&mask, &good_mask);
andnot_mask(mask, compat_mask); slice_andnot_mask(&mask, &compat_mask);
if (mask.low_slices || mask.high_slices) { if (mask.low_slices || !bitmap_empty(mask.high_slices, SLICE_NUM_HIGH)) {
slice_convert(mm, mask, psize); slice_convert(mm, mask, psize);
if (psize > MMU_PAGE_BASE) if (psize > MMU_PAGE_BASE)
on_each_cpu(slice_flush_segments, mm, 1); on_each_cpu(slice_flush_segments, mm, 1);
...@@ -700,7 +738,7 @@ int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, ...@@ -700,7 +738,7 @@ int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
if (psize == MMU_PAGE_64K) { if (psize == MMU_PAGE_64K) {
struct slice_mask compat_mask; struct slice_mask compat_mask;
compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K); compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
or_mask(available, compat_mask); slice_or_mask(&available, &compat_mask);
} }
#endif #endif
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment