Commit dc434e05 authored by Sebastian Andrzej Siewior's avatar Sebastian Andrzej Siewior Committed by Thomas Gleixner

cpu/hotplug: Serialize callback invocations proper

The setup/remove_state/instance() functions in the hotplug core code are
serialized against concurrent CPU hotplug, but unfortunately not serialized
against themself.

As a consequence a concurrent invocation of these function results in
corruption of the callback machinery because two instances try to invoke
callbacks on remote cpus at the same time. This results in missing callback
invocations and initiator threads waiting forever on the completion.

The obvious solution to replace get_cpu_online() with cpu_hotplug_begin()
is not possible because at least one callsite calls into these functions
from a get_online_cpu() locked region.

Extend the protection scope of the cpuhp_state_mutex from solely protecting
the state arrays to cover the callback invocation machinery as well.

Fixes: 5b7aa87e ("cpu/hotplug: Implement setup/removal interface")
Reported-and-tested-by: default avatarBart Van Assche <Bart.VanAssche@sandisk.com>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: hpa@zytor.com
Cc: mingo@kernel.org
Cc: akpm@linux-foundation.org
Cc: torvalds@linux-foundation.org
Link: http://lkml.kernel.org/r/20170314150645.g4tdyoszlcbajmna@linutronix.deSigned-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 4495c08e
...@@ -1335,26 +1335,21 @@ static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, ...@@ -1335,26 +1335,21 @@ static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
struct cpuhp_step *sp; struct cpuhp_step *sp;
int ret = 0; int ret = 0;
mutex_lock(&cpuhp_state_mutex);
if (state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN) { if (state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN) {
ret = cpuhp_reserve_state(state); ret = cpuhp_reserve_state(state);
if (ret < 0) if (ret < 0)
goto out; return ret;
state = ret; state = ret;
} }
sp = cpuhp_get_step(state); sp = cpuhp_get_step(state);
if (name && sp->name) { if (name && sp->name)
ret = -EBUSY; return -EBUSY;
goto out;
}
sp->startup.single = startup; sp->startup.single = startup;
sp->teardown.single = teardown; sp->teardown.single = teardown;
sp->name = name; sp->name = name;
sp->multi_instance = multi_instance; sp->multi_instance = multi_instance;
INIT_HLIST_HEAD(&sp->list); INIT_HLIST_HEAD(&sp->list);
out:
mutex_unlock(&cpuhp_state_mutex);
return ret; return ret;
} }
...@@ -1428,6 +1423,7 @@ int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, ...@@ -1428,6 +1423,7 @@ int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
return -EINVAL; return -EINVAL;
get_online_cpus(); get_online_cpus();
mutex_lock(&cpuhp_state_mutex);
if (!invoke || !sp->startup.multi) if (!invoke || !sp->startup.multi)
goto add_node; goto add_node;
...@@ -1447,16 +1443,14 @@ int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, ...@@ -1447,16 +1443,14 @@ int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
if (ret) { if (ret) {
if (sp->teardown.multi) if (sp->teardown.multi)
cpuhp_rollback_install(cpu, state, node); cpuhp_rollback_install(cpu, state, node);
goto err; goto unlock;
} }
} }
add_node: add_node:
ret = 0; ret = 0;
mutex_lock(&cpuhp_state_mutex);
hlist_add_head(node, &sp->list); hlist_add_head(node, &sp->list);
unlock:
mutex_unlock(&cpuhp_state_mutex); mutex_unlock(&cpuhp_state_mutex);
err:
put_online_cpus(); put_online_cpus();
return ret; return ret;
} }
...@@ -1491,6 +1485,7 @@ int __cpuhp_setup_state(enum cpuhp_state state, ...@@ -1491,6 +1485,7 @@ int __cpuhp_setup_state(enum cpuhp_state state,
return -EINVAL; return -EINVAL;
get_online_cpus(); get_online_cpus();
mutex_lock(&cpuhp_state_mutex);
ret = cpuhp_store_callbacks(state, name, startup, teardown, ret = cpuhp_store_callbacks(state, name, startup, teardown,
multi_instance); multi_instance);
...@@ -1524,6 +1519,7 @@ int __cpuhp_setup_state(enum cpuhp_state state, ...@@ -1524,6 +1519,7 @@ int __cpuhp_setup_state(enum cpuhp_state state,
} }
} }
out: out:
mutex_unlock(&cpuhp_state_mutex);
put_online_cpus(); put_online_cpus();
/* /*
* If the requested state is CPUHP_AP_ONLINE_DYN, return the * If the requested state is CPUHP_AP_ONLINE_DYN, return the
...@@ -1547,6 +1543,8 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state, ...@@ -1547,6 +1543,8 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state,
return -EINVAL; return -EINVAL;
get_online_cpus(); get_online_cpus();
mutex_lock(&cpuhp_state_mutex);
if (!invoke || !cpuhp_get_teardown_cb(state)) if (!invoke || !cpuhp_get_teardown_cb(state))
goto remove; goto remove;
/* /*
...@@ -1563,7 +1561,6 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state, ...@@ -1563,7 +1561,6 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state,
} }
remove: remove:
mutex_lock(&cpuhp_state_mutex);
hlist_del(node); hlist_del(node);
mutex_unlock(&cpuhp_state_mutex); mutex_unlock(&cpuhp_state_mutex);
put_online_cpus(); put_online_cpus();
...@@ -1571,6 +1568,7 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state, ...@@ -1571,6 +1568,7 @@ int __cpuhp_state_remove_instance(enum cpuhp_state state,
return 0; return 0;
} }
EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance); EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
/** /**
* __cpuhp_remove_state - Remove the callbacks for an hotplug machine state * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state
* @state: The state to remove * @state: The state to remove
...@@ -1589,6 +1587,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) ...@@ -1589,6 +1587,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
get_online_cpus(); get_online_cpus();
mutex_lock(&cpuhp_state_mutex);
if (sp->multi_instance) { if (sp->multi_instance) {
WARN(!hlist_empty(&sp->list), WARN(!hlist_empty(&sp->list),
"Error: Removing state %d which has instances left.\n", "Error: Removing state %d which has instances left.\n",
...@@ -1613,6 +1612,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) ...@@ -1613,6 +1612,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
} }
remove: remove:
cpuhp_store_callbacks(state, NULL, NULL, NULL, false); cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
mutex_unlock(&cpuhp_state_mutex);
put_online_cpus(); put_online_cpus();
} }
EXPORT_SYMBOL(__cpuhp_remove_state); EXPORT_SYMBOL(__cpuhp_remove_state);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment