Commit 0a08d470 authored by Lisa Casino's avatar Lisa Casino

slapos/collect: add usrquota

Quotas allow the administrator to limit the space occupied by each user (partition)
When quotas are installed, the collect will no longer use the "du" command
but "repquota" to get the used size for each user.
parent 468b503b
...@@ -70,7 +70,6 @@ download-binary-dir-url = http://shadir.nxdcdn.com ...@@ -70,7 +70,6 @@ download-binary-dir-url = http://shadir.nxdcdn.com
enable = True enable = True
time_cycle = 86400 time_cycle = 86400
pid_folder = /srv/slapgrid/var/run pid_folder = /srv/slapgrid/var/run
use_quota = False
# List of signatures of uploaders we trust: # List of signatures of uploaders we trust:
# Sebastien Robin # Sebastien Robin
......
...@@ -29,11 +29,30 @@ ...@@ -29,11 +29,30 @@
import os import os
import logging import logging
import subprocess
import re
from datetime import datetime, timedelta from datetime import datetime, timedelta
from slapos.collect.snapshot import FolderSizeSnapshot from slapos.collect.snapshot import FolderSizeSnapshot
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def has_quota(path):
command_df = 'findmnt -n -o SOURCE --target %s' % path
process_df = subprocess.Popen(command_df, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
result_df = process_df.communicate()[0]
if process_df.returncode == 0:
device = result_df
command_mount = 'mount | grep %s' % device
process_mount = subprocess.Popen(command_mount, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
result_mount = process_mount.communicate()[0]
if process_mount.returncode == 0:
output = re.split(' |,|\(|\)', result_mount.strip())
if "usrquota" in output:
return True
return False
def get_user_list(config): def get_user_list(config):
nb_user = int(config.get("slapformat", "partition_amount")) nb_user = int(config.get("slapformat", "partition_amount"))
name_prefix = config.get("slapformat", "user_base_name") name_prefix = config.get("slapformat", "user_base_name")
...@@ -41,11 +60,13 @@ def get_user_list(config): ...@@ -41,11 +60,13 @@ def get_user_list(config):
instance_root = config.get("slapos", "instance_root") instance_root = config.get("slapos", "instance_root")
# By default, enable disk snapshot, # By default, enable disk snapshot,
# and set time_cycle to 24hours after the first disk snapshot run # and set time_cycle to 24hours after the first disk snapshot run
use_quota = has_quota(instance_root)
logger.debug("Use of quotas: %s", use_quota)
pid_folder_tmp = instance_root + "/var/run" pid_folder_tmp = instance_root + "/var/run"
disk_snapshot_params = {'enable': True, disk_snapshot_params = {'enable': True,
'time_cycle': 86400, 'time_cycle': 86400,
'pid_folder': pid_folder_tmp, 'pid_folder': pid_folder_tmp,
'use_quota': False} 'use_quota': use_quota}
if config.has_section('collect'): if config.has_section('collect'):
collect_section = dict(config.items("collect")) collect_section = dict(config.items("collect"))
...@@ -53,7 +74,7 @@ def get_user_list(config): ...@@ -53,7 +74,7 @@ def get_user_list(config):
enable = collect_section.get("report_disk_usage", "True").lower() in ('true', 'on', '1'), enable = collect_section.get("report_disk_usage", "True").lower() in ('true', 'on', '1'),
pid_folder = collect_section.get("disk_snapshot_process_pid_foder", pid_folder_tmp), pid_folder = collect_section.get("disk_snapshot_process_pid_foder", pid_folder_tmp),
time_cycle = int(collect_section.get("disk_snapshot_time_cycle", 86400)), time_cycle = int(collect_section.get("disk_snapshot_time_cycle", 86400)),
use_quota = collect_section.get("disk_snapshot_use_quota", "False").lower() in ('true', 'on', '1'), use_quota = collect_section.get("disk_snapshot_use_quota", use_quota).lower() in ('true', 'on', '1'),
) )
user_dict = {name: User(name, path, disk_snapshot_params) user_dict = {name: User(name, path, disk_snapshot_params)
for name, path in [ for name, path in [
...@@ -82,7 +103,7 @@ class User(object): ...@@ -82,7 +103,7 @@ class User(object):
time_cycle = self.disk_snapshot_params.get('time_cycle', 0) time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
database.connect() database.connect()
if time_cycle: if time_cycle and not(self.disk_snapshot_params['use_quota']):
for date_time in database.select(table="folder", columns="date, time", for date_time in database.select(table="folder", columns="date, time",
order='date DESC, time DESC', limit=1, order='date DESC, time DESC', limit=1,
where="partition='%s'" % self.name): where="partition='%s'" % self.name):
...@@ -98,8 +119,8 @@ class User(object): ...@@ -98,8 +119,8 @@ class User(object):
pid_file = self.disk_snapshot_params.get('pid_folder', None) pid_file = self.disk_snapshot_params.get('pid_folder', None)
if pid_file is not None: if pid_file is not None:
pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name) pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
use_quota = self.disk_snapshot_params['use_quota']
disk_snapshot = FolderSizeSnapshot(self.path, pid_file) disk_snapshot = FolderSizeSnapshot(self.path, pid_file, use_quota, self.name)
disk_snapshot.update_folder_size() disk_snapshot.update_folder_size()
# Skeep insert empty partition: size <= 1Mb # Skeep insert empty partition: size <= 1Mb
if disk_snapshot.disk_usage <= 1024.0 and \ if disk_snapshot.disk_usage <= 1024.0 and \
...@@ -107,6 +128,8 @@ class User(object): ...@@ -107,6 +128,8 @@ class User(object):
logger.debug("Disk usage of the partition %s: %s. " logger.debug("Disk usage of the partition %s: %s. "
"Ignoring insertion in the dataset.", self.name, disk_snapshot.disk_usage) "Ignoring insertion in the dataset.", self.name, disk_snapshot.disk_usage)
return return
logger.debug("Disk usage of the partition %s: %s. "
"Insertion in the dataset.", self.name, disk_snapshot.disk_usage)
database.inserFolderSnapshot(self.name, database.inserFolderSnapshot(self.name,
disk_usage=disk_snapshot.get("disk_usage"), disk_usage=disk_snapshot.get("disk_usage"),
insertion_date=collected_date, insertion_date=collected_date,
......
...@@ -80,12 +80,13 @@ class ProcessSnapshot(_Snapshot): ...@@ -80,12 +80,13 @@ class ProcessSnapshot(_Snapshot):
class FolderSizeSnapshot(_Snapshot): class FolderSizeSnapshot(_Snapshot):
"""Calculate partition folder size. """Calculate partition folder size.
""" """
def __init__(self, folder_path, pid_file=None, use_quota=False): def __init__(self, folder_path, pid_file=None, use_quota=False, username=None):
# slapos computer partition size # slapos computer partition size
self.folder_path = folder_path self.folder_path = folder_path
self.pid_file = pid_file self.pid_file = pid_file
self.disk_usage = 0 self.disk_usage = 0
self.use_quota = use_quota self.use_quota = use_quota
self.username = username
def update_folder_size(self): def update_folder_size(self):
"""Return 0 if the process du is still running """Return 0 if the process du is still running
...@@ -114,8 +115,21 @@ class FolderSizeSnapshot(_Snapshot): ...@@ -114,8 +115,21 @@ class FolderSizeSnapshot(_Snapshot):
self.disk_usage += self._getSize('%s/' % extra_path) self.disk_usage += self._getSize('%s/' % extra_path)
def _getSize(self, file_path): def _getSize(self, file_path):
size = 0 size = 0
if self.use_quota and self.username:
command = 'repquota -u /'
process = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
if self.pid_file:
with open(self.pid_file, 'w') as fpid:
pid = fpid.write(str(process.pid))
result = process.communicate()[0]
if process.returncode == 0:
lines = result.strip().split("\n")
for line in lines:
if line.split()[0] == self.username:
size = line.split()[2]
else:
command = 'du -s %s' % file_path command = 'du -s %s' % file_path
process = subprocess.Popen(command, stdout=subprocess.PIPE, process = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True) stderr=subprocess.PIPE, shell=True)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment