Commit e9e9b7ec authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

mm: swap: unify cluster-based and vma-based swap readahead

This patch makes do_swap_page() not need to be aware of two different
swap readahead algorithms.  Just unify cluster-based and vma-based
readahead function call.

Link: http://lkml.kernel.org/r/1509520520-32367-3-git-send-email-minchan@kernel.org
Link: http://lkml.kernel.org/r/20180220085249.151400-3-minchan@kernel.orgSigned-off-by: default avatarMinchan Kim <minchan@kernel.org>
Reviewed-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Huang Ying <ying.huang@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent eaf649eb
...@@ -400,7 +400,6 @@ int generic_swapfile_activate(struct swap_info_struct *, struct file *, ...@@ -400,7 +400,6 @@ int generic_swapfile_activate(struct swap_info_struct *, struct file *,
#define SWAP_ADDRESS_SPACE_SHIFT 14 #define SWAP_ADDRESS_SPACE_SHIFT 14
#define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT) #define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT)
extern struct address_space *swapper_spaces[]; extern struct address_space *swapper_spaces[];
extern bool swap_vma_readahead;
#define swap_address_space(entry) \ #define swap_address_space(entry) \
(&swapper_spaces[swp_type(entry)][swp_offset(entry) \ (&swapper_spaces[swp_type(entry)][swp_offset(entry) \
>> SWAP_ADDRESS_SPACE_SHIFT]) >> SWAP_ADDRESS_SPACE_SHIFT])
...@@ -422,10 +421,10 @@ extern struct page *read_swap_cache_async(swp_entry_t, gfp_t, ...@@ -422,10 +421,10 @@ extern struct page *read_swap_cache_async(swp_entry_t, gfp_t,
extern struct page *__read_swap_cache_async(swp_entry_t, gfp_t, extern struct page *__read_swap_cache_async(swp_entry_t, gfp_t,
struct vm_area_struct *vma, unsigned long addr, struct vm_area_struct *vma, unsigned long addr,
bool *new_page_allocated); bool *new_page_allocated);
extern struct page *swapin_readahead(swp_entry_t, gfp_t, extern struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
struct vm_area_struct *vma, unsigned long addr); struct vm_fault *vmf);
extern struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask, extern struct page *swapin_readahead(swp_entry_t entry, gfp_t flag,
struct vm_fault *vmf); struct vm_fault *vmf);
/* linux/mm/swapfile.c */ /* linux/mm/swapfile.c */
extern atomic_long_t nr_swap_pages; extern atomic_long_t nr_swap_pages;
...@@ -433,11 +432,6 @@ extern long total_swap_pages; ...@@ -433,11 +432,6 @@ extern long total_swap_pages;
extern atomic_t nr_rotate_swap; extern atomic_t nr_rotate_swap;
extern bool has_usable_swap(void); extern bool has_usable_swap(void);
static inline bool swap_use_vma_readahead(void)
{
return READ_ONCE(swap_vma_readahead) && !atomic_read(&nr_rotate_swap);
}
/* Swap 50% full? Release swapcache more aggressively.. */ /* Swap 50% full? Release swapcache more aggressively.. */
static inline bool vm_swap_full(void) static inline bool vm_swap_full(void)
{ {
...@@ -533,19 +527,14 @@ static inline void put_swap_page(struct page *page, swp_entry_t swp) ...@@ -533,19 +527,14 @@ static inline void put_swap_page(struct page *page, swp_entry_t swp)
{ {
} }
static inline struct page *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask, static inline struct page *swap_cluster_readahead(swp_entry_t entry,
struct vm_area_struct *vma, unsigned long addr) gfp_t gfp_mask, struct vm_fault *vmf)
{ {
return NULL; return NULL;
} }
static inline bool swap_use_vma_readahead(void) static inline struct page *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask,
{ struct vm_fault *vmf)
return false;
}
static inline struct page *do_swap_page_readahead(swp_entry_t fentry,
gfp_t gfp_mask, struct vm_fault *vmf)
{ {
return NULL; return NULL;
} }
......
...@@ -2927,7 +2927,8 @@ int do_swap_page(struct vm_fault *vmf) ...@@ -2927,7 +2927,8 @@ int do_swap_page(struct vm_fault *vmf)
if (si->flags & SWP_SYNCHRONOUS_IO && if (si->flags & SWP_SYNCHRONOUS_IO &&
__swap_count(si, entry) == 1) { __swap_count(si, entry) == 1) {
/* skip swapcache */ /* skip swapcache */
page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address); page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
vmf->address);
if (page) { if (page) {
__SetPageLocked(page); __SetPageLocked(page);
__SetPageSwapBacked(page); __SetPageSwapBacked(page);
...@@ -2936,12 +2937,8 @@ int do_swap_page(struct vm_fault *vmf) ...@@ -2936,12 +2937,8 @@ int do_swap_page(struct vm_fault *vmf)
swap_readpage(page, true); swap_readpage(page, true);
} }
} else { } else {
if (swap_use_vma_readahead()) page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
page = do_swap_page_readahead(entry, vmf);
GFP_HIGHUSER_MOVABLE, vmf);
else
page = swapin_readahead(entry,
GFP_HIGHUSER_MOVABLE, vma, vmf->address);
swapcache = page; swapcache = page;
} }
......
...@@ -1422,9 +1422,12 @@ static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, ...@@ -1422,9 +1422,12 @@ static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
{ {
struct vm_area_struct pvma; struct vm_area_struct pvma;
struct page *page; struct page *page;
struct vm_fault vmf;
shmem_pseudo_vma_init(&pvma, info, index); shmem_pseudo_vma_init(&pvma, info, index);
page = swapin_readahead(swap, gfp, &pvma, 0); vmf.vma = &pvma;
vmf.address = 0;
page = swap_cluster_readahead(swap, gfp, &vmf);
shmem_pseudo_vma_destroy(&pvma); shmem_pseudo_vma_destroy(&pvma);
return page; return page;
......
...@@ -38,7 +38,7 @@ static const struct address_space_operations swap_aops = { ...@@ -38,7 +38,7 @@ static const struct address_space_operations swap_aops = {
struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly; struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly;
static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly; static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly;
bool swap_vma_readahead __read_mostly = true; bool enable_vma_readahead __read_mostly = true;
#define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2)
#define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1)
...@@ -322,6 +322,11 @@ void free_pages_and_swap_cache(struct page **pages, int nr) ...@@ -322,6 +322,11 @@ void free_pages_and_swap_cache(struct page **pages, int nr)
release_pages(pagep, nr); release_pages(pagep, nr);
} }
static inline bool swap_use_vma_readahead(void)
{
return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);
}
/* /*
* Lookup a swap entry in the swap cache. A found page will be returned * Lookup a swap entry in the swap cache. A found page will be returned
* unlocked and with its refcount incremented - we rely on the kernel * unlocked and with its refcount incremented - we rely on the kernel
...@@ -544,11 +549,10 @@ static unsigned long swapin_nr_pages(unsigned long offset) ...@@ -544,11 +549,10 @@ static unsigned long swapin_nr_pages(unsigned long offset)
} }
/** /**
* swapin_readahead - swap in pages in hope we need them soon * swap_cluster_readahead - swap in pages in hope we need them soon
* @entry: swap entry of this memory * @entry: swap entry of this memory
* @gfp_mask: memory allocation flags * @gfp_mask: memory allocation flags
* @vma: user vma this address belongs to * @vmf: fault information
* @addr: target address for mempolicy
* *
* Returns the struct page for entry and addr, after queueing swapin. * Returns the struct page for entry and addr, after queueing swapin.
* *
...@@ -560,10 +564,10 @@ static unsigned long swapin_nr_pages(unsigned long offset) ...@@ -560,10 +564,10 @@ static unsigned long swapin_nr_pages(unsigned long offset)
* This has been extended to use the NUMA policies from the mm triggering * This has been extended to use the NUMA policies from the mm triggering
* the readahead. * the readahead.
* *
* Caller must hold down_read on the vma->vm_mm if vma is not NULL. * Caller must hold down_read on the vma->vm_mm if vmf->vma is not NULL.
*/ */
struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
struct vm_area_struct *vma, unsigned long addr) struct vm_fault *vmf)
{ {
struct page *page; struct page *page;
unsigned long entry_offset = swp_offset(entry); unsigned long entry_offset = swp_offset(entry);
...@@ -573,6 +577,8 @@ struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, ...@@ -573,6 +577,8 @@ struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
struct swap_info_struct *si = swp_swap_info(entry); struct swap_info_struct *si = swp_swap_info(entry);
struct blk_plug plug; struct blk_plug plug;
bool do_poll = true, page_allocated; bool do_poll = true, page_allocated;
struct vm_area_struct *vma = vmf->vma;
unsigned long addr = vmf->address;
mask = swapin_nr_pages(offset) - 1; mask = swapin_nr_pages(offset) - 1;
if (!mask) if (!mask)
...@@ -727,7 +733,7 @@ static void swap_ra_info(struct vm_fault *vmf, ...@@ -727,7 +733,7 @@ static void swap_ra_info(struct vm_fault *vmf,
pte_unmap(orig_pte); pte_unmap(orig_pte);
} }
struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask, struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask,
struct vm_fault *vmf) struct vm_fault *vmf)
{ {
struct blk_plug plug; struct blk_plug plug;
...@@ -774,20 +780,40 @@ struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask, ...@@ -774,20 +780,40 @@ struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
ra_info.win == 1); ra_info.win == 1);
} }
/**
* swapin_readahead - swap in pages in hope we need them soon
* @entry: swap entry of this memory
* @gfp_mask: memory allocation flags
* @vmf: fault information
*
* Returns the struct page for entry and addr, after queueing swapin.
*
* It's a main entry function for swap readahead. By the configuration,
* it will read ahead blocks by cluster-based(ie, physical disk based)
* or vma-based(ie, virtual address based on faulty address) readahead.
*/
struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
struct vm_fault *vmf)
{
return swap_use_vma_readahead() ?
swap_vma_readahead(entry, gfp_mask, vmf) :
swap_cluster_readahead(entry, gfp_mask, vmf);
}
#ifdef CONFIG_SYSFS #ifdef CONFIG_SYSFS
static ssize_t vma_ra_enabled_show(struct kobject *kobj, static ssize_t vma_ra_enabled_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf) struct kobj_attribute *attr, char *buf)
{ {
return sprintf(buf, "%s\n", swap_vma_readahead ? "true" : "false"); return sprintf(buf, "%s\n", enable_vma_readahead ? "true" : "false");
} }
static ssize_t vma_ra_enabled_store(struct kobject *kobj, static ssize_t vma_ra_enabled_store(struct kobject *kobj,
struct kobj_attribute *attr, struct kobj_attribute *attr,
const char *buf, size_t count) const char *buf, size_t count)
{ {
if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1)) if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1))
swap_vma_readahead = true; enable_vma_readahead = true;
else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1)) else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1))
swap_vma_readahead = false; enable_vma_readahead = false;
else else
return -EINVAL; return -EINVAL;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment