slapos/collect: Enable disk usage by default

slapos.cfg.example

@@ -66,6 +66,12 @@ download-binary-dir-url = http://shadir.nxdcdn.com
 #shadir-cert-file = /etc/opt/slapos/shacache/shacache.cert
 #shadir-key-file = /etc/opt/slapos/shacache/shacache.key
 
+[collect]
+enable = True 
+time_cycle = 86400
+pid_folder = /srv/slapgrid/var/run
+use_quota = False
+
 # List of signatures of uploaders we trust:
 #   Sebastien Robin
 #   Kazuhiko Shiozaki

slapos/cli/collect.py

@@ -31,6 +31,8 @@ from slapos.collect import do_collect
 from slapos.cli.command import must_be_root
 from slapos.cli.config import ConfigCommand
 
+import logging
+
 class CollectCommand(ConfigCommand):
     """
     Collect system consumption and data and store.
@@ -44,4 +46,5 @@ class CollectCommand(ConfigCommand):
     @must_be_root
     def take_action(self, args):
         configp = self.fetch_config(args)
-        do_collect(configp)
+        logger = self.app.log
+        do_collect(logger, configp)

slapos/collect/__init__.py

@@ -71,7 +71,7 @@ def current_state(user_dict):
   for i, process in enumerate(process_list):
     yield build_snapshot(process)
 
-def do_collect(conf):
+def do_collect(logger, conf):
   """
   Main function
   The idea here is to poll system every so many seconds
@@ -80,6 +80,7 @@ def do_collect(conf):
     Each user object is a dict, indexed on timestamp. We add every snapshot
     matching the user so that we get informations for each users
   """
+  logger.info('Collecting data...')
   try:
     collected_date, collected_time = _get_time()
     user_dict = get_user_list(conf)
@@ -94,16 +95,20 @@ def do_collect(conf):
     if conf.has_option("slapos", "collect_cache"):
       days_to_preserve = conf.getint("slapos", "collect_cache")
     log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
+    
+    logger.debug("Log directory: %s", log_directory)
     mkdir_p(log_directory, 0o755)
     
     consumption_report_directory = "%s/var/consumption-report" % \
-                                        conf.get("slapos", "instance_root") 
+                                        conf.get("slapos", "instance_root")
     mkdir_p(consumption_report_directory, 0o755)
+    logger.debug("Consumption report directory: %s", consumption_report_directory)
 
     xml_report_directory = "%s/var/xml_report/%s" % \
                     (conf.get("slapos", "instance_root"), 
                      conf.get("slapos", "computer_id"))
     mkdir_p(xml_report_directory, 0o755)
+    logger.debug("XML report directory: %s", xml_report_directory)
 
     if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
       os.chmod(log_directory, 0o755)    
@@ -113,11 +118,12 @@ def do_collect(conf):
     if conf.has_option("slapformat", "computer_model_id"):
       computer_model_id = conf.get("slapformat", 
                                   "computer_model_id")
- 
     else:
       computer_model_id = "no_model"
+    logger.debug("Computer model id: %s", computer_model_id)
 
     uptime = _get_uptime()
+
     if conf.has_option("slapformat", "heating_sensor_id"):
       heating_sensor_id = conf.get("slapformat", 
                                   "heating_sensor_id")
@@ -130,21 +136,33 @@ def do_collect(conf):
     else:
       heating_sensor_id = "no_sensor"
       test_heating = False
+    logger.debug("Heating sensor id: %s", heating_sensor_id)
 
+    logger.info("Inserting computer information into database...")
     computer = Computer(ComputerSnapshot(model_id=computer_model_id, 
                                      sensor_id = heating_sensor_id,
                                      test_heating=test_heating))
 
+    # Insert computer's data
     computer.save(database, collected_date, collected_time)
 
+    logger.info("Done.")
+    logger.info("Inserting user information into database...")
+
+    # Insert TABLE user + TABLE folder
     for user in user_dict.values():
       user.save(database, collected_date, collected_time)
-    
+
+    logger.info("Done.")
+    logger.info("Writing csv, XML and JSON files...")
+    # Write a csv with dumped data in the log_directory
     SystemCSVReporterDumper(database).dump(log_directory)
     RawCSVDumper(database).dump(log_directory)
+
+    # Write xml files
     consumption_report = ConsumptionReport(
                       computer_id=conf.get("slapos", "computer_id"), 
-                      user_list=user_dict, 
+                      user_list=user_dict,
                       database=database,
                       location=consumption_report_directory)
     
@@ -156,16 +174,23 @@ def do_collect(conf):
       if report_file is not None:
         shutil.copy(report_file, xml_report_directory)
 
+    # write json
     partition_report = PartitionReport(
             database=database,
             user_list=user_dict)
 
     partition_report.buildJSONMonitorReport()
+
+    # Put dumped csv in a current_date.tar.gz
     compressLogFolder(log_directory)
+    logger.info("Done.")
 
     # Drop older entries already reported
     database.garbageCollect(days_to_preserve)
 
+    logger.info("Finished collecting.")
+    logger.info('=' * 80)
+
   except AccessDenied:
-    print("You HAVE TO execute this script with root permission.")
+    logger.error("You HAVE TO execute this script with root permission.")
 

slapos/collect/entity.py

@@ -28,9 +28,12 @@
 ##############################################################################
 
 import os
+import logging
 from datetime import datetime, timedelta
 from slapos.collect.snapshot import FolderSizeSnapshot
 
+logger = logging.getLogger(__name__)
+
 def get_user_list(config):
   nb_user = int(config.get("slapformat", "partition_amount"))
   name_prefix = config.get("slapformat", "user_base_name")
@@ -38,14 +41,19 @@ def get_user_list(config):
   instance_root = config.get("slapos", "instance_root")
   # By default, enable disk snapshot,
   # and set time_cycle to 24hours after the first disk snapshot run
-  disk_snapshot_params = {'enable': False, 'time_cycle': 86400}
+  pid_folder_tmp = instance_root + "/var/run"
+  disk_snapshot_params = {'enable': True, 
+                          'time_cycle': 86400,
+                          'pid_folder': pid_folder_tmp, 
+                          'use_quota': False}
+
   if config.has_section('collect'):
     collect_section = dict(config.items("collect"))
     disk_snapshot_params = dict(
-      enable=collect_section.get("report_disk_usage", "False").lower() in ('true', 'on', '1'),
-      pid_folder=collect_section.get("disk_snapshot_process_pid_foder", None),
-      time_cycle=int(collect_section.get("disk_snapshot_time_cycle", 86400)),
-      use_quota=collect_section.get("disk_snapshot_use_quota", "True").lower() in ('true', 'on', '1'),
+      enable = collect_section.get("report_disk_usage", "True").lower() in ('true', 'on', '1'),
+      pid_folder = collect_section.get("disk_snapshot_process_pid_foder", pid_folder_tmp),
+      time_cycle = int(collect_section.get("disk_snapshot_time_cycle", 86400)),
+      use_quota = collect_section.get("disk_snapshot_use_quota", "False").lower() in ('true', 'on', '1'),
     )
   user_dict = {name: User(name, path, disk_snapshot_params)
       for name, path in [
@@ -72,6 +80,7 @@ class User(object):
   def _insertDiskSnapShot(self, database, collected_date, collected_time):
     if self.disk_snapshot_params['enable']:
       time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
+
       database.connect()
       if time_cycle:
         for date_time in database.select(table="folder", columns="date, time",
@@ -79,18 +88,24 @@ class User(object):
                                     where="partition='%s'" % self.name):
           latest_date = datetime.strptime('%s %s' % date_time,
                                           "%Y-%m-%d %H:%M:%S")
-          if (datetime.utcnow() - latest_date).seconds < time_cycle:
+          time_spent = (datetime.utcnow() - latest_date).total_seconds()
+          if time_spent < time_cycle:
             # wait the time cycle
+            logger.info("Time cycle is not over (%s seconds remaining). No computation of " 
+                        "disk usage on the partition %s.", time_cycle - time_spent, self.name)
             return
           break
       pid_file = self.disk_snapshot_params.get('pid_folder', None)
       if pid_file is not None:
         pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
+
       disk_snapshot = FolderSizeSnapshot(self.path, pid_file)
       disk_snapshot.update_folder_size()
       # Skeep insert empty partition: size <= 1Mb
       if disk_snapshot.disk_usage <= 1024.0 and \
                       not self.disk_snapshot_params.get('testing', False):
+        logger.debug("Disk usage of the partition %s: %s. "
+                  "Ignoring insertion in the dataset.", self.name, disk_snapshot.disk_usage)
         return
       database.inserFolderSnapshot(self.name,
                 disk_usage=disk_snapshot.get("disk_usage"),
@@ -102,7 +117,6 @@ class User(object):
   def save(self, database, collected_date, collected_time):
     """ Insert collected data on user collector """
     database.connect()
-    snapshot_counter = len(self.snapshot_list)
     for snapshot_item in self.snapshot_list:
       snapshot_item.update_cpu_percent()
       database.insertUserSnapshot(self.name,

slapos/collect/snapshot.py

@@ -31,6 +31,7 @@ from __future__ import print_function
 import psutil
 import os
 import subprocess
+import logging
 from .temperature import collectComputerTemperature, launchTemperatureTest
 
 from .temperature.heating import get_contribution_ratio
@@ -39,6 +40,8 @@ import six
 
 MEASURE_INTERVAL = 5
 
+logger = logging.getLogger(__name__)
+
 class _Snapshot(object):
   def get(self, property, default=None):
     return getattr(self, property, default)
@@ -97,18 +100,21 @@ class FolderSizeSnapshot(_Snapshot):
           except OSError:
             pass
           else:
+            logger.warning("Process %s still in progress. Try later.", pid)
             return
 
     self.disk_usage = self._getSize(self.folder_path)
     # If extra disk added to partition
     data_dir = os.path.join(self.folder_path, 'DATA')
     if os.path.exists(data_dir):
+      logger.debug("Extra disk added to the partition")
       for filename in os.listdir(data_dir):
         extra_path = os.path.join(data_dir, filename)
         if os.path.islink(extra_path) and os.path.isdir('%s/' % extra_path):
           self.disk_usage += self._getSize('%s/' % extra_path)
 
   def _getSize(self, file_path):
+
     size = 0
     command = 'du -s %s' % file_path
     process = subprocess.Popen(command, stdout=subprocess.PIPE, 
@@ -119,6 +125,7 @@ class FolderSizeSnapshot(_Snapshot):
     result = process.communicate()[0]
     if process.returncode == 0:
       size, _  = result.strip().split()
+
     return float(size)
 
 class SystemSnapshot(_Snapshot):
@@ -161,8 +168,7 @@ class HeatingContributionSnapshot(_Snapshot):
     
     result = launchTemperatureTest(sensor_id)
     if result is None:
-      print("Impossible to test sensor: %s " % sensor_id)
-      
+      logger.warning("Impossible to test sensor: %s", sensor_id)
 
     initial_temperature, final_temperature, duration = result