Commit 8bb366ab by Tomáš Peterka

[manager.cpuset] Split CPU cores half-half between shared and exclusive

1 parent d041e861
# coding: utf-8
import logging
import math
import os
import os.path
import pwd
......@@ -45,20 +46,39 @@ class Manager(object):
logger.warning("CPUSet Manager cannot format computer because cgroups do not exist.")
for cpu in self._cpu_id_list():
def create_cpuset_subtree(name, cpu_id_list):
"""Create root for cpuset with per-CPU folders."""
if not cpu_id_list:
logger.warning("Too less cores - cannot reserve any for \"{}\"".format(name))
root_path = self._prepare_folder(
os.path.join(self.cpuset_path, name))
with open(root_path + "/cpuset.cpus", "wt") as fx:
# this cgroup manages all `name` cpus
if len(cpu_id_list) == 1:
fx.write("{:d}-{:d}".format(cpu_id_list[0], cpu_id_list[-1]))
create_cpuset_subtree("shared", self._shared_cpu_id_list())
create_cpuset_subtree("exclusive", self._exclusive_cpu_id_list())
# separate CPUs in exclusive set
exclusive_cpu_path = os.path.join(self.cpuset_path, "exclusive")
write_file("1", exclusive_cpu_path + "/cpuset.cpu_exclusive") # exclusive
write_file("0", exclusive_cpu_path + "/cpuset.mems") # it doesn't work without that
for cpu in self._exclusive_cpu_id_list():
cpu_path = self._prepare_folder(
os.path.join(self.cpuset_path, "cpu" + str(cpu)))
with open(cpu_path + "/cpuset.cpus", "wt") as fx:
fx.write(str(cpu)) # this cgroup manages only this cpu
with open(cpu_path + "/cpuset.cpu_exclusive", "wt") as fx:
fx.write("1") # manages it exclusively
with open(cpu_path + "/cpuset.mems", "wt") as fx:
fx.write("0") # it doesn't work without that
os.path.join(self.cpuset_path, "exclusive", "cpu" + str(cpu)))
write_file(str(cpu), cpu_path + "/cpuset.cpus") # manage only this cpu
write_file("1", cpu_path + "/cpuset.cpu_exclusive") # exclusive
write_file("0", cpu_path + "/cpuset.mems") # it doesn't work without that
def instance(self, partition):
"""Control runtime state of the computer."""
if not os.path.exists(os.path.join(self.cpuset_path, "cpu0")):
if not os.path.exists(os.path.join(self.cpuset_path, "shared")):
# check whether the computer was formatted
logger.warning("CGROUP's CPUSET Manager cannot update computer because it is not cpuset-formatted.")
......@@ -73,15 +93,13 @@ class Manager(object):
uid, gid = partition.getUserGroupId()
uname = pwd.getpwuid(uid).pw_name
if uname not in power_user_list:
logger.warning("User {} not allowed to modify cpuset! "
logger.warning("{} is not allowed to modify cpuset! "
"Allowed users are in {} option in config file.".format(
uname, self.config_power_user_option))
# prepare paths to tasks file for all and per-cpu
tasks_file = os.path.join(self.cpuset_path, "tasks")
cpu_tasks_file_list = [os.path.join(cpu_folder, "tasks")
for cpu_folder in self._cpu_folder_list()]
# Gather exclusive CPU usage map {username: set[cpu_id]}
# We do not need to gather that since we have no limits yet
......@@ -93,13 +111,14 @@ class Manager(object):
# process = psutil.Process(pid)
# cpu_usage[process.username()].add(cpu_id)
# Move all PIDs from the pool of all CPUs onto the first exclusive CPU.
# Move all PIDs from the pool of all CPUs onto shared CPUs.
running_list = sorted(list(map(int, read_file(tasks_file).split())), reverse=True)
first_cpu = self._cpu_id_list()[0]
shared_tasks_path = os.path.join(self.cpuset_path, "shared", "tasks")
exclusive_tasks_path = os.path.join(self.cpuset_path, "exclusive", "tasks")
success_set, refused_set = set(), set()
for pid in running_list:
self._move_task(pid, first_cpu)
write_file("{:d}\n".format(pid), shared_tasks_path, mode=self.task_write_mode)
except IOError as e:
......@@ -108,37 +127,40 @@ class Manager(object):
"Suceeded in moving {:d} PIDs {!s}\n".format(
len(refused_set), refused_set, len(success_set), success_set))
cpu_folder_list = self._cpu_folder_list()
generic_cpu_path = cpu_folder_list[0]
exclusive_cpu_path_list = cpu_folder_list[1:]
# Gather all running PIDs for filtering out stale PIDs
running_pid_set = set(running_list)
running_pid_set.update(map(int, read_file(cpu_tasks_file_list[0]).split()))
running_pid_set.update(map(int, read_file(shared_tasks_path).split()))
# gather already exclusively running PIDs
# Gather already exclusively running PIDs
exclusive_pid_set = set()
for cpu_tasks_file in cpu_tasks_file_list[1:]:
for cpu_tasks_file in [exclusive_tasks_path, ] + self._exclusive_cpu_tasks_list():
exclusive_pid_set.update(map(int, read_file(cpu_tasks_file).split()))
# Move processes to their demanded exclusive CPUs
with open(request_file, "rt") as fi:
# take such PIDs which are either really running or are not already exclusive
request_pid_list = [int(pid) for pid in
if int(pid) in running_pid_set or int(pid) not in exclusive_pid_set]
with open(request_file, "wt") as fo:
fo.write("") # empty file (we will write back only PIDs which weren't moved)
for request_pid in request_pid_list:
# move processes to their demanded exclusive CPUs
request_pid_list = map(int, read_file(request_file).split())
# empty the request file (we will write back only PIDs which weren't moved)
write_file("", request_file)
# take such PIDs which are either really running or are not already exclusive
for request_pid in filter(lambda pid: pid in running_pid_set and pid not in exclusive_pid_set, request_pid_list):
assigned_cpu = self._move_to_exclusive_cpu(request_pid)
if assigned_cpu < 0:
# if no exclusive CPU was assigned - write the PID back and try other time
with open(request_file, "at") as fo:
fo.write(str(request_pid) + "\n")
write_file("{:d}\n".format(request_pid), request_file, mode="at")
def _cpu_folder_list(self):
def _exclusive_cpu_tasks_list(self):
"""Return list of folders for exclusive cpu cores."""
return [os.path.join(self.cpuset_path, "cpu" + str(cpu_id))
for cpu_id in self._cpu_id_list()]
return [os.path.join(self.cpuset_path, "exclusive", "cpu" + str(cpu_id), "tasks")
for cpu_id in self._exclusive_cpu_id_list()]
def _shared_cpu_id_list(self):
"""Return list of shared CPU core IDs."""
cpu_id_list = self._cpu_id_list()
return cpu_id_list[:int(math.ceil(len(cpu_id_list) / 2))]
def _exclusive_cpu_id_list(self):
"""Return list of exclusive CPU core IDs."""
cpu_id_list = self._cpu_id_list()
return cpu_id_list[int(math.ceil(len(cpu_id_list) / 2)):]
def _cpu_id_list(self):
"""Extract IDs of available CPUs and return them as a list.
......@@ -162,12 +184,12 @@ class Manager(object):
:return: int, cpu_id of used CPU, -1 if placement was not possible
exclusive_cpu_list = self._cpu_id_list()[1:]
exclusive_cpu_list = self._exclusive_cpu_id_list()
for exclusive_cpu in exclusive_cpu_list:
# gather tasks assigned to current exclusive CPU
task_path = os.path.join(self.cpuset_path, "cpu" + str(exclusive_cpu), "tasks")
with open(task_path, "rt") as fi:
task_list =
task_path = os.path.join(
self.cpuset_path, "exclusive", "cpu" + str(exclusive_cpu), "tasks")
task_list = read_file(task_path).split()
if len(task_list) > 0:
continue # skip occupied CPUs
return self._move_task(pid, exclusive_cpu)[1]
......@@ -179,8 +201,11 @@ class Manager(object):
cpu_mode can be "performance" or "powersave"
known_cpu_mode_list = ("performance", "powersave")
with open(os.path.join(self.cpuset_path, "cpu" + str(cpu_id), "tasks"), self.task_write_mode) as fo:
fo.write(str(pid) + "\n")
if cpu_id not in self._exclusive_cpu_id_list():
raise ValueError("Cannot assign to cpu{:d} because it is shared".format(cpu_id))
# write the PID into concrete tasks file
cpu_tasks = os.path.join(self.cpuset_path, "exclusive", "cpu" + str(cpu_id), "tasks")
write_file("{:d}\n".format(pid), cpu_tasks, mode=self.task_write_mode)
# set the core to `cpu_mode`
scaling_governor_file = "/sys/devices/system/cpu/cpu{:d}/cpufreq/scaling_governor".format(cpu_id)
if os.path.exists(scaling_governor_file):
......@@ -216,5 +241,9 @@ def read_file(path, mode="rt"):
def write_file(content, path, mode="wt"):
with open(path, mode) as fo:
\ No newline at end of file
with open(path, mode) as fo:
except IOError as e:
logger.error("Cannot write to {}".format(path))
\ No newline at end of file
......@@ -744,31 +744,49 @@ class TestComputerWithCPUSet(SlapformatMixin):
self.assertEqual([0]._cpu_id_list(), self.cpu_list)
# This should created per-cpu groups and move all tasks in CPU pool into cpu0, alter_user=False)
# Test files creation for exclusive CPUs
for cpu_id in self.cpu_list:
cpu_n_path = os.path.join(self.cpuset_path, "cpu" + str(cpu_id))
# First half of CPUs is shared
shared_cpuset_path = os.path.join(self.cpuset_path, "shared")
for cpu_id in self.cpu_list[:2]:
os.path.join(shared_cpuset_path, "cpu" + str(cpu_id))),
"Specific cpu<N> doesn't make sense in shared cpuset.")
# The second half of CPUs most be ready for exclusive
exclusive_cpuset_path = os.path.join(self.cpuset_path, "exclusive")
for cpu_id in self.cpu_list[2:]:
cpu_n_path = os.path.join(exclusive_cpuset_path, "cpu" + str(cpu_id))
self.assertEqual(str(cpu_id), file_content(os.path.join(cpu_n_path, "cpuset.cpus")))
self.assertEqual("1", file_content(os.path.join(cpu_n_path, "cpuset.cpu_exclusive")))
if cpu_id > 0:
self.assertEqual("", file_content(os.path.join(cpu_n_path, "tasks")))
for cpu_id in self.cpu_list[:2]:
os.path.join(exclusive_cpuset_path, "cpu" + str(cpu_id))),
"Shared CPU cannot appear in EXCLUSIVE set.")
# Test moving tasks from generic core to private core
# request PID 1001 to be moved to its private CPU
request_file_path = os.path.join([0].path,
file_write("1001\n", request_file_path)
# ask exclusivity for existing and non-existing PID
file_write("1001\n9999\n", request_file_path)
# Simulate slapos instance call to perform the actual movement[0].instance(
# Simulate cgroup behaviour - empty tasks in the pool
file_write("", os.path.join(self.cpuset_path, "tasks"))
# Test that format moved all PIDs from CPU pool into CPU0
tasks_at_cpu0 = file_content(os.path.join(self.cpuset_path, "cpu0", "tasks")).split()
self.assertIn("1000", tasks_at_cpu0)
# Test that format moved all PIDs from CPU pool into shared pool
shared_task_list = file_content(os.path.join(self.cpuset_path, "shared", "tasks")).split()
self.assertIn("1000", shared_task_list)
self.assertIn("1002", shared_task_list)
# test if the moving suceeded into any provate CPUS (id>0)
self.assertTrue(any("1001" in file_content(exclusive_task)
for exclusive_task in glob.glob(os.path.join(self.cpuset_path, "cpu[1-9]", "tasks"))))
self.assertIn("1002", tasks_at_cpu0)
exclusive_tasks_path_list = [
os.path.join(self.cpuset_path, "exclusive", "tasks"),
] + glob.glob(os.path.join(self.cpuset_path, "exclusive", "cpu[1-9]", "tasks"))
self.assertTrue(any("1001" in file_content(exclusive_tasks_path)
for exclusive_tasks_path in exclusive_tasks_path_list))
self.assertFalse(any("9999" in file_content(exclusive_tasks_path)
for exclusive_tasks_path in exclusive_tasks_path_list))
# slapformat should remove successfully moved PIDs from the .slapos-cpu-exclusive file
self.assertEqual("", file_content(request_file_path).strip())
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!