Commit bd233f53 authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds

mm, page_alloc: use static global work_struct for draining per-cpu pages

As suggested by Vlastimil Babka and Tejun Heo, this patch uses a static
work_struct to co-ordinate the draining of per-cpu pages on the
workqueue.  Only one task can drain at a time but this is better than
the previous scheme that allowed multiple tasks to send IPIs at a time.

One consideration is whether parallel requests should synchronise
against each other.  This patch does not synchronise for a global drain
as the common case for such callers is expected to be multiple parallel
direct reclaimers competing for pages when the watermark is close to
min.  Draining the per-cpu list is unlikely to make much progress and
serialising the drain is of dubious merit.  Drains are synchonrised for
callers such as memory hotplug and CMA that care about the drain being
complete when the function returns.

Link: http://lkml.kernel.org/r/20170125083038.rzb5f43nptmk7aed@techsingularity.netSigned-off-by: default avatarMel Gorman <mgorman@techsingularity.net>
Suggested-by: default avatarTejun Heo <tj@kernel.org>
Suggested-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarHillf Danton <hillf.zj@alibaba-inc.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 51047820
...@@ -92,6 +92,10 @@ EXPORT_PER_CPU_SYMBOL(_numa_mem_); ...@@ -92,6 +92,10 @@ EXPORT_PER_CPU_SYMBOL(_numa_mem_);
int _node_numa_mem_[MAX_NUMNODES]; int _node_numa_mem_[MAX_NUMNODES];
#endif #endif
/* work_structs for global per-cpu drains */
DEFINE_MUTEX(pcpu_drain_mutex);
DEFINE_PER_CPU(struct work_struct, pcpu_drain);
#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
volatile unsigned long latent_entropy __latent_entropy; volatile unsigned long latent_entropy __latent_entropy;
EXPORT_SYMBOL(latent_entropy); EXPORT_SYMBOL(latent_entropy);
...@@ -2360,7 +2364,6 @@ static void drain_local_pages_wq(struct work_struct *work) ...@@ -2360,7 +2364,6 @@ static void drain_local_pages_wq(struct work_struct *work)
*/ */
void drain_all_pages(struct zone *zone) void drain_all_pages(struct zone *zone)
{ {
struct work_struct __percpu *works;
int cpu; int cpu;
/* /*
...@@ -2373,7 +2376,16 @@ void drain_all_pages(struct zone *zone) ...@@ -2373,7 +2376,16 @@ void drain_all_pages(struct zone *zone)
if (current->flags & PF_WQ_WORKER) if (current->flags & PF_WQ_WORKER)
return; return;
works = alloc_percpu_gfp(struct work_struct, GFP_ATOMIC); /*
* Do not drain if one is already in progress unless it's specific to
* a zone. Such callers are primarily CMA and memory hotplug and need
* the drain to be complete when the call returns.
*/
if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
if (!zone)
return;
mutex_lock(&pcpu_drain_mutex);
}
/* /*
* We don't care about racing with CPU hotplug event * We don't care about racing with CPU hotplug event
...@@ -2406,23 +2418,15 @@ void drain_all_pages(struct zone *zone) ...@@ -2406,23 +2418,15 @@ void drain_all_pages(struct zone *zone)
cpumask_clear_cpu(cpu, &cpus_with_pcps); cpumask_clear_cpu(cpu, &cpus_with_pcps);
} }
if (works) { for_each_cpu(cpu, &cpus_with_pcps) {
for_each_cpu(cpu, &cpus_with_pcps) { struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
struct work_struct *work = per_cpu_ptr(works, cpu); INIT_WORK(work, drain_local_pages_wq);
INIT_WORK(work, drain_local_pages_wq); schedule_work_on(cpu, work);
schedule_work_on(cpu, work);
}
for_each_cpu(cpu, &cpus_with_pcps)
flush_work(per_cpu_ptr(works, cpu));
} else {
for_each_cpu(cpu, &cpus_with_pcps) {
struct work_struct work;
INIT_WORK(&work, drain_local_pages_wq);
schedule_work_on(cpu, &work);
flush_work(&work);
}
} }
for_each_cpu(cpu, &cpus_with_pcps)
flush_work(per_cpu_ptr(&pcpu_drain, cpu));
mutex_unlock(&pcpu_drain_mutex);
} }
#ifdef CONFIG_HIBERNATION #ifdef CONFIG_HIBERNATION
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment