Commit ae3066bd authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/mm/slice: implement slice_check_range_fits

Rather than build slice masks from a range then use that to check for
fit in a candidate mask, implement slice_check_range_fits that checks
if a range fits in a mask directly.

This allows several structures to be removed from stacks, and also we
don't expect a huge range in a lot of these cases, so building and
comparing a full mask is going to be more expensive than testing just
one or two bits of the range.

On POWER8, this increases vfork+exec+exit performance by 0.3%
and reduces time to mmap+munmap a 64kB page by 5%.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 5709f7cf
...@@ -174,26 +174,36 @@ static struct slice_mask *slice_mask_for_size(struct mm_struct *mm, int psize) ...@@ -174,26 +174,36 @@ static struct slice_mask *slice_mask_for_size(struct mm_struct *mm, int psize)
#error "Must define the slice masks for page sizes supported by the platform" #error "Must define the slice masks for page sizes supported by the platform"
#endif #endif
static int slice_check_fit(struct mm_struct *mm, static bool slice_check_range_fits(struct mm_struct *mm,
const struct slice_mask *mask, const struct slice_mask *available,
const struct slice_mask *available) unsigned long start, unsigned long len)
{ {
DECLARE_BITMAP(result, SLICE_NUM_HIGH); unsigned long end = start + len - 1;
/* u64 low_slices = 0;
* Make sure we just do bit compare only to the max
* addr limit and not the full bit map size.
*/
unsigned long slice_count = GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit);
if (!SLICE_NUM_HIGH) if (start < SLICE_LOW_TOP) {
return (mask->low_slices & available->low_slices) == unsigned long mend = min(end,
mask->low_slices; (unsigned long)(SLICE_LOW_TOP - 1));
low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
- (1u << GET_LOW_SLICE_INDEX(start));
}
if ((low_slices & available->low_slices) != low_slices)
return false;
if (SLICE_NUM_HIGH && ((start + len) > SLICE_LOW_TOP)) {
unsigned long start_index = GET_HIGH_SLICE_INDEX(start);
unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT));
unsigned long count = GET_HIGH_SLICE_INDEX(align_end) - start_index;
unsigned long i;
bitmap_and(result, mask->high_slices, for (i = start_index; i < start_index + count; i++) {
available->high_slices, slice_count); if (!test_bit(i, available->high_slices))
return false;
}
}
return (mask->low_slices & available->low_slices) == mask->low_slices && return true;
bitmap_equal(result, mask->high_slices, slice_count);
} }
static void slice_flush_segments(void *parm) static void slice_flush_segments(void *parm)
...@@ -558,14 +568,10 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -558,14 +568,10 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
/* First check hint if it's valid or if we have MAP_FIXED */ /* First check hint if it's valid or if we have MAP_FIXED */
if (addr != 0 || fixed) { if (addr != 0 || fixed) {
/* Build a mask for the requested range */
slice_range_to_mask(addr, len, &mask);
slice_print_mask(" mask", &mask);
/* Check if we fit in the good mask. If we do, we just return, /* Check if we fit in the good mask. If we do, we just return,
* nothing else to do * nothing else to do
*/ */
if (slice_check_fit(mm, &mask, &good_mask)) { if (slice_check_range_fits(mm, &good_mask, addr, len)) {
slice_dbg(" fits good !\n"); slice_dbg(" fits good !\n");
return addr; return addr;
} }
...@@ -591,11 +597,12 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len, ...@@ -591,11 +597,12 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
slice_or_mask(&potential_mask, &good_mask); slice_or_mask(&potential_mask, &good_mask);
slice_print_mask(" potential", &potential_mask); slice_print_mask(" potential", &potential_mask);
if ((addr != 0 || fixed) && if (addr != 0 || fixed) {
slice_check_fit(mm, &mask, &potential_mask)) { if (slice_check_range_fits(mm, &potential_mask, addr, len)) {
slice_dbg(" fits potential !\n"); slice_dbg(" fits potential !\n");
goto convert; goto convert;
} }
}
/* If we have MAP_FIXED and failed the above steps, then error out */ /* If we have MAP_FIXED and failed the above steps, then error out */
if (fixed) if (fixed)
...@@ -772,13 +779,12 @@ void slice_set_range_psize(struct mm_struct *mm, unsigned long start, ...@@ -772,13 +779,12 @@ void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
unsigned long len) unsigned long len)
{ {
struct slice_mask mask, available; struct slice_mask available;
unsigned int psize = mm->context.user_psize; unsigned int psize = mm->context.user_psize;
if (radix_enabled()) if (radix_enabled())
return 0; return 0;
slice_range_to_mask(addr, len, &mask);
available = *slice_mask_for_size(mm, psize); available = *slice_mask_for_size(mm, psize);
#ifdef CONFIG_PPC_64K_PAGES #ifdef CONFIG_PPC_64K_PAGES
/* We need to account for 4k slices too */ /* We need to account for 4k slices too */
...@@ -795,6 +801,6 @@ int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, ...@@ -795,6 +801,6 @@ int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
slice_print_mask(" mask", &mask); slice_print_mask(" mask", &mask);
slice_print_mask(" available", &available); slice_print_mask(" available", &available);
#endif #endif
return !slice_check_fit(mm, &mask, &available); return !slice_check_range_fits(mm, &available, addr, len);
} }
#endif #endif
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment