slapos/collect: add usrquota

Quotas allow the administrator to limit the space occupied by each user (partition)
When quotas are installed, the collect will no longer use the "du" command
but "repquota" to get the used size for each user.

slapos.cfg.example

@@ -70,7 +70,6 @@ download-binary-dir-url = http://shadir.nxdcdn.com
 enable = True 
 time_cycle = 86400
 pid_folder = /srv/slapgrid/var/run
-use_quota = False
 
 # List of signatures of uploaders we trust:
 #   Sebastien Robin

slapos/collect/entity.py

@@ -29,11 +29,30 @@
 
 import os
 import logging
+import subprocess
+import re
 from datetime import datetime, timedelta
 from slapos.collect.snapshot import FolderSizeSnapshot
 
 logger = logging.getLogger(__name__)
 
+def has_quota(path):
+  command_df = 'findmnt -n -o SOURCE --target %s' % path
+  process_df = subprocess.Popen(command_df, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, shell=True)
+  result_df = process_df.communicate()[0]
+  if process_df.returncode == 0:
+    device = result_df
+    command_mount = 'mount | grep %s' % device
+    process_mount = subprocess.Popen(command_mount, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, shell=True)
+    result_mount = process_mount.communicate()[0]
+    if process_mount.returncode == 0:
+      output = re.split(' |,|\(|\)', result_mount.strip())
+      if "usrquota" in output:
+        return True
+  return False
+
 def get_user_list(config):
   nb_user = int(config.get("slapformat", "partition_amount"))
   name_prefix = config.get("slapformat", "user_base_name")
@@ -41,11 +60,13 @@ def get_user_list(config):
   instance_root = config.get("slapos", "instance_root")
   # By default, enable disk snapshot,
   # and set time_cycle to 24hours after the first disk snapshot run
+  use_quota = has_quota(instance_root)
+  logger.debug("Use of quotas: %s", use_quota)
   pid_folder_tmp = instance_root + "/var/run"
   disk_snapshot_params = {'enable': True, 
                           'time_cycle': 86400,
                           'pid_folder': pid_folder_tmp, 
-                          'use_quota': False}
+                          'use_quota': use_quota}
 
   if config.has_section('collect'):
     collect_section = dict(config.items("collect"))
@@ -53,7 +74,7 @@ def get_user_list(config):
       enable = collect_section.get("report_disk_usage", "True").lower() in ('true', 'on', '1'),
       pid_folder = collect_section.get("disk_snapshot_process_pid_foder", pid_folder_tmp),
       time_cycle = int(collect_section.get("disk_snapshot_time_cycle", 86400)),
-      use_quota = collect_section.get("disk_snapshot_use_quota", "False").lower() in ('true', 'on', '1'),
+      use_quota = collect_section.get("disk_snapshot_use_quota", use_quota).lower() in ('true', 'on', '1'),
     )
   user_dict = {name: User(name, path, disk_snapshot_params)
       for name, path in [
@@ -82,7 +103,7 @@ class User(object):
       time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
 
       database.connect()
-      if time_cycle:
+      if time_cycle and not(self.disk_snapshot_params['use_quota']):
         for date_time in database.select(table="folder", columns="date, time",
                                     order='date DESC, time DESC', limit=1,
                                     where="partition='%s'" % self.name):
@@ -98,8 +119,8 @@ class User(object):
       pid_file = self.disk_snapshot_params.get('pid_folder', None)
       if pid_file is not None:
         pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
-
-      disk_snapshot = FolderSizeSnapshot(self.path, pid_file)
+      use_quota = self.disk_snapshot_params['use_quota']
+      disk_snapshot = FolderSizeSnapshot(self.path, pid_file, use_quota, self.name)
       disk_snapshot.update_folder_size()
       # Skeep insert empty partition: size <= 1Mb
       if disk_snapshot.disk_usage <= 1024.0 and \
@@ -107,6 +128,8 @@ class User(object):
         logger.debug("Disk usage of the partition %s: %s. "
                   "Ignoring insertion in the dataset.", self.name, disk_snapshot.disk_usage)
         return
+      logger.debug("Disk usage of the partition %s: %s. "
+                  "Insertion in the dataset.", self.name, disk_snapshot.disk_usage)
       database.inserFolderSnapshot(self.name,
                 disk_usage=disk_snapshot.get("disk_usage"),
                 insertion_date=collected_date,

slapos/collect/snapshot.py

@@ -80,12 +80,13 @@ class ProcessSnapshot(_Snapshot):
 class FolderSizeSnapshot(_Snapshot):
   """Calculate partition folder size. 
   """
-  def __init__(self, folder_path, pid_file=None, use_quota=False):
+  def __init__(self, folder_path, pid_file=None, use_quota=False, username=None):
     # slapos computer partition size
     self.folder_path = folder_path
     self.pid_file = pid_file
     self.disk_usage = 0
     self.use_quota = use_quota
+    self.username = username
 
   def update_folder_size(self):
     """Return 0 if the process du is still running
@@ -114,17 +115,30 @@ class FolderSizeSnapshot(_Snapshot):
           self.disk_usage += self._getSize('%s/' % extra_path)
 
   def _getSize(self, file_path):
-
     size = 0
-    command = 'du -s %s' % file_path
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, 
-                              stderr=subprocess.PIPE, shell=True)
-    if self.pid_file:
-      with open(self.pid_file, 'w') as fpid:
-        pid = fpid.write(str(process.pid))
-    result = process.communicate()[0]
-    if process.returncode == 0:
-      size, _  = result.strip().split()
+    if self.use_quota and self.username:
+      command = 'repquota -u /'
+      process = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE, shell=True)
+      if self.pid_file:
+        with open(self.pid_file, 'w') as fpid:
+          pid = fpid.write(str(process.pid))
+      result = process.communicate()[0]
+      if process.returncode == 0:
+        lines = result.strip().split("\n")
+        for line in lines:
+          if line.split()[0] == self.username:
+            size = line.split()[2]
+    else:
+      command = 'du -s %s' % file_path
+      process = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, shell=True)
+      if self.pid_file:
+        with open(self.pid_file, 'w') as fpid:
+          pid = fpid.write(str(process.pid))
+      result = process.communicate()[0]
+      if process.returncode == 0:
+        size, _  = result.strip().split()
 
     return float(size)