Add partition size snapshot to slapos.collect
Partition folder snapshot use du to calculate folder size and return value in Mb. This snapshot is enabled by default and will calculate size of each partition every 24hours. Partition with size < 1Mb are not are not reported (considered as empty). The behavior can be changed using some configurations in slapos.cfg: [collect] report_disk_usage = True disk_snapshot_process_pid_foder = /srv/slapgrid/var/run disk_snapshot_time_cycle = 86400
Showing
| ... | @@ -27,14 +27,27 @@ | ... | @@ -27,14 +27,27 @@ |
| # | # | ||
| ############################################################################## | ############################################################################## | ||
| import os | |||
| from datetime import datetime, timedelta | |||
| from slapos.collect.snapshot import FolderSizeSnapshot | |||
| def get_user_list(config): | def get_user_list(config): | ||
| nb_user = int(config.get("slapformat", "partition_amount")) | nb_user = int(config.get("slapformat", "partition_amount")) | ||
| name_prefix = config.get("slapformat", "user_base_name") | name_prefix = config.get("slapformat", "user_base_name") | ||
| path_prefix = config.get("slapformat", "partition_base_name") | path_prefix = config.get("slapformat", "partition_base_name") | ||
| instance_root = config.get("slapos", "instance_root") | instance_root = config.get("slapos", "instance_root") | ||
| user_dict = {name: User(name, path) | # By default, enable disk snapshot, | ||
| # and set time_cycle to 24hours after the first disk snapshot run | |||
| disk_snapshot_params = {'enable': True, 'time_cycle': 86400} | |||
| if config.has_section('collect'): | |||
| collect_section = dict(config.items("collect")) | |||
| disk_snapshot_params = dict( | |||
| enable=eval(collect_section.get("report_disk_usage", "True")), | |||
|
|||
| pid_folder=collect_section.get("disk_snapshot_process_pid_foder", None), | |||
| time_cycle=int(collect_section.get("disk_snapshot_time_cycle", 86400)), | |||
| use_quota=eval(collect_section.get("disk_snapshot_use_quota", "True")) | |||
| ) | |||
| user_dict = {name: User(name, path, disk_snapshot_params) | |||
| for name, path in [ | for name, path in [ | ||
| ( | ( | ||
| "%s%s" % (name_prefix, nb), | "%s%s" % (name_prefix, nb), | ||
| ... | @@ -47,17 +60,53 @@ def get_user_list(config): | ... | @@ -47,17 +60,53 @@ def get_user_list(config): |
| return user_dict | return user_dict | ||
| class User(object): | class User(object): | ||
| def __init__(self, name, path): | def __init__(self, name, path, disk_snapshot_params={}): | ||
| self.name = str(name) | self.name = str(name) | ||
| self.path = str(path) | self.path = str(path) | ||
| self.disk_snapshot_params = disk_snapshot_params | |||
| self.snapshot_list = [] | self.snapshot_list = [] | ||
| def append(self, value): | def append(self, value): | ||
| self.snapshot_list.append(value) | self.snapshot_list.append(value) | ||
| def _insertDiskSnapShot(self, database, collected_date, collected_time): | |||
| if self.disk_snapshot_params['enable']: | |||
| time_cycle = self.disk_snapshot_params.get('time_cycle', 0) | |||
| database.connect() | |||
| if time_cycle: | |||
| order = 'date DESC, time DESC' | |||
| limit = 1 | |||
| query = database.select(table="folder", columns="date, time", | |||
| order=order, limit=limit, | |||
| where="partition=%s" % self.name) | |||
| query_result = zip(*query) | |||
| if len(query_result): | |||
| date, time = (query_result[0][0], query_result[1][0]) | |||
| latest_date = datetime.strptime('%s %s' % (date, time), | |||
| "%Y-%m-%d %H:%M:%S") | |||
| if (datetime.now() - latest_date).seconds < time_cycle: | |||
| # wait the time cycle | |||
| return | |||
| pid_file = self.disk_snapshot_params.get('pid_folder', None) | |||
| if pid_file is not None: | |||
| pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name) | |||
| disk_snapshot = FolderSizeSnapshot(self.path, pid_file) | |||
| disk_snapshot.update_folder_size() | |||
| # Skeep insert empty partition: size <= 1Mb | |||
| if disk_snapshot.disk_usage <= 1.0 and \ | |||
| not self.disk_snapshot_params.get('testing', False): | |||
| return | |||
| database.inserFolderSnapshot(self.name, | |||
| disk_usage=disk_snapshot.get("disk_usage"), | |||
| insertion_date=collected_date, | |||
| insertion_time=collected_time) | |||
| database.commit() | |||
| database.close() | |||
| def save(self, database, collected_date, collected_time): | def save(self, database, collected_date, collected_time): | ||
| """ Insert collected data on user collector """ | """ Insert collected data on user collector """ | ||
| database.connect() | database.connect() | ||
| snapshot_counter = len(self.snapshot_list) | |||
| for snapshot_item in self.snapshot_list: | for snapshot_item in self.snapshot_list: | ||
| snapshot_item.update_cpu_percent() | snapshot_item.update_cpu_percent() | ||
| database.insertUserSnapshot(self.name, | database.insertUserSnapshot(self.name, | ||
| ... | @@ -74,6 +123,9 @@ class User(object): | ... | @@ -74,6 +123,9 @@ class User(object): |
| insertion_time=collected_time) | insertion_time=collected_time) | ||
| database.commit() | database.commit() | ||
| database.close() | database.close() | ||
| # Inser disk snapshot in a new transaction, it can take long | |||
| self._insertDiskSnapShot(database, collected_date, collected_time) | |||
| class Computer(dict): | class Computer(dict): | ||
| ... | ... | ||
