Skip to content

S
slapos.core
Commit 324177d5 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

Merge branch 'collect'

parents dec95dfc 8f7ca6db
Expand all Show whitespace changes
Inline Side-by-side
Showing with 1249 additions and 904 deletions
setup.py
View file @ 324177d5
...@@ -48,6 +48,7 @@ setup(name=name, ...@@ -48,6 +48,7 @@ setup(name=name,
'netifaces', # to fetch information about network devices 'netifaces', # to fetch information about network devices
'setuptools', # namespaces 'setuptools', # namespaces
'supervisor', # slapgrid uses supervisor to manage processes 'supervisor', # slapgrid uses supervisor to manage processes
'psutil',
'xml_marshaller>=0.9.3', # to unmarshall/marshall python objects to/from 'xml_marshaller>=0.9.3', # to unmarshall/marshall python objects to/from
# XML # XML
'zope.interface', # slap library implementes interfaces 'zope.interface', # slap library implementes interfaces
...@@ -72,17 +73,7 @@ setup(name=name, ...@@ -72,17 +73,7 @@ setup(name=name,
entry_points={ entry_points={
'console_scripts': [ 'console_scripts': [
'slapos-watchdog = slapos.grid.watchdog:main', 'slapos-watchdog = slapos.grid.watchdog:main',
'slapproxy = slapos.cli_legacy.proxy_start:main',
'slapos = slapos.cli.entry:main', 'slapos = slapos.cli.entry:main',
# Deprecated entry points
'slapconsole = slapos.cli_legacy.console:console',
'slapformat = slapos.cli_legacy.format:main',
'slapgrid-sr = slapos.cli_legacy.slapgrid:runSoftwareRelease',
'slapgrid-cp = slapos.cli_legacy.slapgrid:runComputerPartition',
'slapgrid-ur = slapos.cli_legacy.slapgrid:runUsageReport',
'slapgrid-supervisorctl = slapos.cli_legacy.svcbackend:supervisorctl',
'slapgrid-supervisord = slapos.cli_legacy.svcbackend:supervisord',
'bang = slapos.cli_legacy.bang:main',
], ],
'slapos.cli': [ 'slapos.cli': [
# Utilities # Utilities
...@@ -102,6 +93,7 @@ setup(name=name, ...@@ -102,6 +93,7 @@ setup(name=name,
'node software = slapos.cli.slapgrid:SoftwareCommand', 'node software = slapos.cli.slapgrid:SoftwareCommand',
'node instance = slapos.cli.slapgrid:InstanceCommand', 'node instance = slapos.cli.slapgrid:InstanceCommand',
'node boot = slapos.cli.boot:BootCommand', 'node boot = slapos.cli.boot:BootCommand',
'node collect = slapos.cli.collect:CollectCommand',
# SlapOS client commands # SlapOS client commands
'console = slapos.cli.console:ConsoleCommand', 'console = slapos.cli.console:ConsoleCommand',
'configure local = slapos.cli.configure_local:ConfigureLocalCommand', 'configure local = slapos.cli.configure_local:ConfigureLocalCommand',
......
slapos/cli/collect.py 0 → 100644
View file @ 324177d5
# -*- coding: utf-8 -*-
import subprocess
from time import sleep
import socket
import glob
import os
from slapos.collect import do_collect
from slapos.cli.command import must_be_root
from slapos.cli.entry import SlapOSApp
from slapos.cli.config import ConfigCommand
class CollectCommand(ConfigCommand):
"""
Collect system consumption and data and store.
"""
command_group = 'node'
def get_parser(self, prog_name):
ap = super(CollectCommand, self).get_parser(prog_name)
return ap
@must_be_root
def take_action(self, args):
configp = self.fetch_config(args)
do_collect(configp)
slapos/cli/register.py
View file @ 324177d5
...@@ -174,10 +174,10 @@ def save_former_config(conf): ...@@ -174,10 +174,10 @@ def save_former_config(conf):
def fetch_configuration_template(): def fetch_configuration_template():
req = requests.get('http://git.erp5.org/gitweb/slapos.core.git/blob_plain/HEAD:/slapos.cfg.example') template_arg_list = (__name__.split('.')[0], 'slapos.cfg.example')
req.raise_for_status() with pkg_resources.resource_stream(*template_arg_list) as fout:
return req.text slapos_node_configuration_template = fout.read()
return slapos_node_configuration_template
def slapconfig(conf): def slapconfig(conf):
"""Base Function to configure slapos in /etc/opt/slapos""" """Base Function to configure slapos in /etc/opt/slapos"""
...@@ -200,13 +200,6 @@ def slapconfig(conf): ...@@ -200,13 +200,6 @@ def slapconfig(conf):
if not dry_run: if not dry_run:
os.mkdir(slap_conf_dir, 0o711) os.mkdir(slap_conf_dir, 0o711)
# Hack that should be removed when we switch to re6st
# Force start of vpn
openvpn_needed_file = os.path.join(slap_conf_dir, 'openvpn-needed')
if not os.path.exists(openvpn_needed_file):
if not dry_run:
open(openvpn_needed_file, 'w').write('')
user_certificate_repository_path = os.path.join(slap_conf_dir, 'ssl') user_certificate_repository_path = os.path.join(slap_conf_dir, 'ssl')
if not os.path.exists(user_certificate_repository_path): if not os.path.exists(user_certificate_repository_path):
conf.logger.info('Creating directory: %s', user_certificate_repository_path) conf.logger.info('Creating directory: %s', user_certificate_repository_path)
...@@ -344,5 +337,5 @@ def do_register(conf): ...@@ -344,5 +337,5 @@ def do_register(conf):
slapconfig(conf) slapconfig(conf)
conf.logger.info('Node has successfully been configured as %s.', COMP) conf.logger.info('Node has successfully been configured as %s.', COMP)
conf.logger.info('Now please invoke /usr/sbin/slapos-start on your site.') conf.logger.info('Now please invoke slapos node boot on your site.')
return 0 return 0
slapos/cli_legacy/bang.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import argparse
import ConfigParser
from slapos.bang import do_bang
def main(*args):
ap = argparse.ArgumentParser()
ap.add_argument('-m', '--message', default='', help='Message for bang.')
ap.add_argument('configuration_file', type=argparse.FileType(),
help='SlapOS configuration file.')
if args:
args = ap.parse_args(list(args))
else:
args = ap.parse_args()
configp = ConfigParser.SafeConfigParser()
configp.readfp(args.configuration_file)
do_bang(configp, args.message)
slapos/cli_legacy/console.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
import argparse
import os
import textwrap
from slapos.client import ClientConfig, init, do_console
from slapos.cli_legacy.util import get_config_parser
def console():
description = textwrap.dedent("""\
slapconsole allows you interact with slap API. You can play with the global
"slap" object and with the global "request" method.
examples :
>>> # Request instance
>>> request(kvm, "myuniquekvm")
>>> # Request software installation on owned computer
>>> supply(kvm, "mycomputer")
>>> # Fetch instance informations on already launched instance
>>> request(kvm, "myuniquekvm").getConnectionParameter("url")""")
ap = argparse.ArgumentParser(description=description,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('-u', '--master_url',
default=None,
help='Url of SlapOS Master to use.')
ap.add_argument('-k', '--key_file',
help="SSL Authorisation key file.")
ap.add_argument('-c', '--cert_file',
help="SSL Authorisation certificate file.")
ap.add_argument('configuration_file',
help='path to slapos.cfg')
args = ap.parse_args()
if not os.path.isfile(args.configuration_file):
ap.error("%s: Not found or not a regular file." % args.configuration_file)
configp = get_config_parser(args.configuration_file)
conf = ClientConfig(args, configp)
local = init(conf)
do_console(local)
slapos/cli_legacy/format.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import argparse
import ConfigParser
import logging
import sys
import os
from slapos.format import FormatConfig, UsageError, tracing_monkeypatch, do_format
def main(*args):
"Run default configuration."
ap = argparse.ArgumentParser()
ap.add_argument('-x', '--computer_xml',
help="Path to file with computer's XML. If does not exists, will be created",
default=None)
ap.add_argument('--computer_json',
help="Path to a JSON version of the computer's XML (for development only).",
default=None)
ap.add_argument('-l', '--log_file',
help="The path to the log file used by the script.")
ap.add_argument('-i', '--input_definition_file',
help="Path to file to read definition of computer instead of "
"declaration. Using definition file allows to disable "
"'discovery' of machine services and allows to define computer "
"configuration in fully controlled manner.")
ap.add_argument('-o', '--output_definition_file',
help="Path to file to write definition of computer from "
"declaration.")
ap.add_argument('-n', '--dry_run',
help="Don't actually do anything.",
default=False,
action="store_true")
ap.add_argument('-v', '--verbose',
default=False,
action="store_true",
help="Verbose output.")
# the console option is actually ignored and not used anymore.
ap.add_argument('-c', '--console',
default=False,
action="store_true",
help="Console output.")
ap.add_argument('--alter_user',
choices=['True', 'False'],
help="Shall slapformat alter user database [default: True]")
ap.add_argument('--alter_network',
choices=['True', 'False'],
help="Shall slapformat alter network configuration [default: True]")
ap.add_argument('--now',
help="Launch slapformat without delay",
default=False,
action="store_true")
ap.add_argument('configuration_file',
help='path to slapos.cfg')
if args:
args = ap.parse_args(list(args))
else:
args = ap.parse_args()
logger = logging.getLogger("slapformat")
logger.addHandler(logging.StreamHandler())
if args.verbose:
logger.setLevel(logging.DEBUG)
logger.debug("Verbose mode enabled.")
else:
logger.setLevel(logging.INFO)
conf = FormatConfig(logger=logger)
configp = ConfigParser.SafeConfigParser()
if configp.read(args.configuration_file) != [args.configuration_file]:
raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
conf.mergeConfig(args, configp)
if conf.log_file:
if not os.path.isdir(os.path.dirname(conf.log_file)):
# fallback to console only if directory for logs does not exists and
# continue to run
raise ValueError('Please create directory %r to store %r log file' % (
os.path.dirname(conf.log_file), conf.log_file))
else:
file_handler = logging.FileHandler(conf.log_file)
file_handler.setFormatter(logging.Formatter("%(asctime)s - "
"%(name)s - %(levelname)s - %(message)s"))
conf.logger.addHandler(file_handler)
conf.logger.info('Configured logging to file %r' % conf.log_file)
try:
conf.setConfig()
except UsageError as exc:
sys.stderr.write(exc.message + '\n')
sys.stderr.write("For help use --help\n")
sys.exit(1)
tracing_monkeypatch(conf)
try:
do_format(conf=conf)
except:
conf.logger.exception('Uncaught exception:')
raise
slapos/cli_legacy/proxy_start.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
# vim: set et sts=2:
import argparse
import ConfigParser
import logging
import os
import sys
from slapos.proxy import ProxyConfig, do_proxy
class UsageError(Exception):
pass
def main():
ap = argparse.ArgumentParser()
ap.add_argument('-l', '--log_file',
help='The path to the log file used by the script.')
ap.add_argument('-v', '--verbose',
action='store_true',
help='Verbose output.')
# XXX not used anymore, deprecated
ap.add_argument('-c', '--console',
action='store_true',
help='Console output.')
ap.add_argument('-u', '--database-uri',
help='URI for sqlite database')
ap.add_argument('configuration_file',
help='path to slapos.cfg')
args = ap.parse_args()
logger = logging.getLogger('slapproxy')
logger.addHandler(logging.StreamHandler())
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
conf = ProxyConfig(logger=logger)
configp = ConfigParser.SafeConfigParser()
if configp.read(args.configuration_file) != [args.configuration_file]:
raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
conf.mergeConfig(args, configp)
if conf.log_file:
if not os.path.isdir(os.path.dirname(conf.log_file)):
raise ValueError('Please create directory %r to store %r log file' % (
os.path.dirname(conf.log_file), conf.log_file))
file_handler = logging.FileHandler(conf.log_file)
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
logger.addHandler(file_handler)
logger.info('Configured logging to file %r' % conf.log_file)
conf.setConfig()
try:
do_proxy(conf=conf)
return_code = 0
except SystemExit as err:
return_code = err
sys.exit(return_code)
slapos/cli_legacy/slapgrid.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
# vim: set et sts=2:
import argparse
import ConfigParser
import logging
import sys
from slapos.grid.utils import setRunning, setFinished
from slapos.grid.slapgrid import (merged_options, check_missing_parameters,
check_missing_files, random_delay, create_slapgrid_object)
def parse_arguments(*argument_tuple):
"""Parse arguments and return options dictionary merged with the config file."""
ap = argparse.ArgumentParser()
ap.add_argument('--instance-root',
help='The instance root directory location.')
ap.add_argument('--software-root',
help='The software_root directory location.')
ap.add_argument('--master-url',
help='The master server URL. Mandatory.')
ap.add_argument('--computer-id',
help='The computer id defined in the server.')
ap.add_argument('--supervisord-socket',
help='The socket supervisor will use.')
ap.add_argument('--supervisord-configuration-path',
help='The location where supervisord configuration will be stored.')
ap.add_argument('--buildout', default=None,
help='Location of buildout binary.')
ap.add_argument('--pidfile',
help='The location where pidfile will be created.')
ap.add_argument('--logfile',
help='The location where slapgrid logfile will be created.')
ap.add_argument('--key_file',
help='SSL Authorisation key file.')
ap.add_argument('--cert_file',
help='SSL Authorisation certificate file.')
ap.add_argument('--signature_private_key_file',
help='Signature private key file.')
ap.add_argument('--master_ca_file',
help='Root certificate of SlapOS master key.')
ap.add_argument('--certificate_repository_path',
help='Path to directory where downloaded certificates would be stored.')
ap.add_argument('-v', '--verbose', action='store_true',
help='Be verbose.')
ap.add_argument('--maximum-periodicity', type=int, default=None,
help='Periodicity at which buildout should be run in instance.')
ap.add_argument('--promise-timeout', type=int, default=3,
help='Promise timeout in seconds.')
ap.add_argument('--now', action='store_true',
help='Launch slapgrid without delay. Default behavior.')
ap.add_argument('--all', action='store_true',
help='Launch slapgrid to process all Software Releases '
'and/or Computer Partitions.')
ap.add_argument('--only-sr',
help='Force the update of a single software release (use url hash), '
'even if is already installed. This option will make all other '
'sofware releases be ignored.')
ap.add_argument('--only-cp',
help='Update a single or a list of computer partitions '
'(ie.:slappartX, slappartY), '
'this option will make all other computer partitions be ignored.')
ap.add_argument('configuration_file', type=argparse.FileType(),
help='SlapOS configuration file.')
# Deprecated options
ap.add_argument('-c', '--console', action='store_true',
help="Deprecated, doesn't do anything.")
ap.add_argument('--develop', action='store_true',
help='Deprecated, same as --all.')
ap.add_argument('--only_sr',
help='Deprecated, same as --only-sr.')
ap.add_argument('--only_cp',
help='Deprecated, same as --only-cp.')
ap.add_argument('--maximal_delay',
help='Deprecated. Will only work from configuration file in the future.')
if not argument_tuple:
args = ap.parse_args()
else:
args = ap.parse_args(list(argument_tuple))
return args
def setup_logger(options):
logger = logging.getLogger(__name__)
if options.get('logfile'):
handler = logging.FileHandler(options['logfile'])
else:
handler = logging.StreamHandler()
if options['verbose']:
handler.setLevel(logging.DEBUG)
else:
handler.setLevel(logging.INFO)
formatter = logging.Formatter(fmt='%(asctime)s %(name)-18s: '
'%(levelname)-8s %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
"""Returns a new instance of slapgrid.Slapgrid created with argument+config parameters.
Also returns the pidfile path, and configures logger.
"""
args = parse_arguments(*argument_tuple)
configp = ConfigParser.SafeConfigParser()
configp.readfp(args.configuration_file)
options = merged_options(args, configp)
logger = setup_logger(options)
check_missing_parameters(options)
check_missing_files(options)
random_delay(options, logger=logger)
slapgrid_object = create_slapgrid_object(options, logger=logger)
return slapgrid_object, options.get('pidfile')
def realRun(argument_tuple, method):
slapgrid_object, pidfile = parseArgumentTupleAndReturnSlapgridObject(*argument_tuple)
if pidfile:
setRunning(logger=slapgrid_object.logger, pidfile=pidfile)
try:
return getattr(slapgrid_object, method)()
finally:
if pidfile:
setFinished(pidfile)
def runSoftwareRelease(*argument_tuple):
"""Hook for entry point to process Software Releases"""
sys.exit(realRun(argument_tuple, 'processSoftwareReleaseList'))
def runComputerPartition(*argument_tuple):
"""Hook for entry point to process Computer Partitions"""
sys.exit(realRun(argument_tuple, 'processComputerPartitionList'))
def runUsageReport(*argument_tuple):
"""Hook for entry point to process Usage Reports"""
sys.exit(realRun(argument_tuple, 'agregateAndSendUsage'))
slapos/cli_legacy/svcbackend.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
# vim: set et sts=2:
##############################################################################
#
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
import os
from optparse import OptionParser
import ConfigParser
from slapos.grid.svcbackend import launchSupervisord
def getOptionDict(*argument_tuple):
usage = """
Typical usage:
* %prog CONFIGURATION_FILE [arguments passed to supervisor]
""".strip()
parser = OptionParser(usage=usage)
# Parses arguments
if argument_tuple:
(argument_option_instance, argument_list) = parser.parse_args(list(argument_tuple))
else:
# No arguments given to entry point : we parse sys.argv.
(argument_option_instance, argument_list) = parser.parse_args()
if not argument_list:
parser.error("Configuration file is obligatory. Consult documentation by calling with -h.")
configuration_file = argument_list[0]
if not os.path.exists(configuration_file):
parser.error("Could not read configuration file : %s" % configuration_file)
slapgrid_configuration = ConfigParser.SafeConfigParser()
slapgrid_configuration.read(configuration_file)
# Merges the two dictionnaries
option_dict = dict(slapgrid_configuration.items("slapos"))
# Supervisord configuration location
option_dict.setdefault('supervisord_configuration_path',
os.path.join(option_dict['instance_root'], 'etc', 'supervisord.conf'))
# Supervisord socket
option_dict.setdefault('supervisord_socket',
os.path.join(option_dict['instance_root'], 'supervisord.socket'))
return option_dict, argument_list[1:]
def supervisorctl(*argument_tuple):
logger = logging.getLogger('SVCBackend')
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
logger.addHandler(handler)
option_dict, args = getOptionDict(*argument_tuple)
import supervisor.supervisorctl
launchSupervisord(socket=option_dict['supervisord_socket'],
configuration_file=option_dict['supervisord_configuration_path'],
logger=logger)
supervisor.supervisorctl.main(args=['-c', option_dict['supervisord_configuration_path']] + args)
def supervisord(*argument_tuple):
logger = logging.getLogger('SVCBackend')
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
logger.addHandler(handler)
option_dict, _ = getOptionDict(*argument_tuple)
launchSupervisord(socket=option_dict['supervisord_socket'],
configuration_file=option_dict['supervisord_configuration_path'],
logger=logger)
slapos/cli_legacy/util.py deleted 100644 → 0
View file @ dec95dfc
# -*- coding: utf-8 -*-
import ConfigParser
import os
def get_config_parser(path):
configp = ConfigParser.SafeConfigParser()
path = os.path.expanduser(path)
if not os.path.isfile(path):
raise OSError('Specified configuration file %s does not exist. Exiting.' % path)
configp.read(path)
return configp
slapos/collect/__init__.py 0 → 100644
View file @ 324177d5
from psutil import process_iter, NoSuchProcess, AccessDenied
from time import time, sleep, strftime
from slapos.collect.db import Database
from slapos.util import mkdir_p
# Local import
from snapshot import ProcessSnapshot, SystemSnapshot, ComputerSnapshot
from slapos.collect.reporter import SystemJSONReporterDumper, \
RawCSVDumper, \
SystemCSVReporterDumper
from entity import get_user_list, Computer
def _get_time():
return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ")
def build_snapshot(proc):
try:
return ProcessSnapshot(proc)
except NoSuchProcess:
return None
def current_state(user_dict):
"""
Iterator used to apply build_snapshot(...) on every single relevant process.
A process is considered relevant if its user matches our user list, i.e.
its user is a slapos user
"""
process_list = [p for p in process_iter() if p.username() in user_dict]
for i, process in enumerate(process_list):
yield build_snapshot(process)
def do_collect(conf):
"""
Main function
The idea here is to poll system every so many seconds
For each poll, we get a list of Snapshots, holding informations about
processes. We iterate over that list to store datas on a per user basis:
Each user object is a dict, indexed on timestamp. We add every snapshot
matching the user so that we get informations for each users
"""
try:
collected_date, collected_time = _get_time()
user_dict = get_user_list(conf)
try:
for snapshot in current_state(user_dict):
if snapshot:
user_dict[snapshot.username].append(snapshot)
except (KeyboardInterrupt, SystemExit, NoSuchProcess):
raise
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory)
database = Database(log_directory)
computer = Computer(ComputerSnapshot())
computer.save(database, collected_date, collected_time)
for user in user_dict.values():
user.save(database, collected_date, collected_time)
SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory)
except AccessDenied:
print "You HAVE TO execute this script with root permission."
slapos/collect/db.py 0 → 100644
View file @ 324177d5
This diff is collapsed.
slapos/collect/entity.py 0 → 100644
View file @ 324177d5
def get_user_list(config):
nb_user = int(config.get("slapformat", "partition_amount"))
name_prefix = config.get("slapformat", "user_base_name")
path_prefix = config.get("slapformat", "partition_base_name")
instance_root = config.get("slapos", "instance_root")
user_dict = {name: User(name, path)
for name, path in [
(
"%s%s" % (name_prefix, nb),
"%s/%s%s" % (instance_root, path_prefix, nb)
) for nb in range(nb_user)
]
}
#user_dict['root'] = User("root", "/opt/slapgrid")
return user_dict
class User(object):
def __init__(self, name, path):
self.name = str(name)
self.path = str(path)
self.snapshot_list = []
def append(self, value):
self.snapshot_list.append(value)
def save(self, database, collected_date, collected_time):
""" Insert collected data on user collector """
database.connect()
for snapshot_item in self.snapshot_list:
database.insertUserSnapshot(self.name,
pid=snapshot_item.get("pid"),
process=snapshot_item.get("process"),
cpu_percent=snapshot_item.get("cpu_percent"),
cpu_time=snapshot_item.get("cpu_time"),
cpu_num_threads=snapshot_item.get("cpu_num_threads"),
memory_percent=snapshot_item.get("memory_percent"),
memory_rss=snapshot_item.get("memory_rss"),
io_rw_counter=snapshot_item.get("io_rw_counter"),
io_cycles_counter=snapshot_item.get("io_cycles_counter"),
insertion_date=collected_date,
insertion_time=collected_time)
database.commit()
database.close()
class Computer(dict):
def __init__(self, computer_snapshot):
self.computer_snapshot = computer_snapshot
def save(self, database, collected_date, collected_time):
database.connect()
self._save_computer_snapshot(database, collected_date, collected_time)
self._save_system_snapshot(database, collected_date, collected_time)
self._save_disk_partition_snapshot(database, collected_date, collected_time)
database.commit()
database.close()
def _save_computer_snapshot(self, database, collected_date, collected_time):
partition_list = ";".join(["%s=%s" % (x,y) for x,y in \
self.computer_snapshot.get("partition_list")])
database.insertComputerSnapshot(
cpu_num_core=self.computer_snapshot.get("cpu_num_core"),
cpu_frequency=self.computer_snapshot.get("cpu_frequency"),
cpu_type=self.computer_snapshot.get("cpu_type"),
memory_size=self.computer_snapshot.get("memory_size"),
memory_type=self.computer_snapshot.get("memory_type"),
partition_list=partition_list,
insertion_date=collected_date,
insertion_time=collected_time)
def _save_system_snapshot(self, database, collected_date, collected_time):
snapshot = self.computer_snapshot.get("system_snapshot")
database.insertSystemSnapshot(
loadavg=snapshot.get("load"),
cpu_percent=snapshot.get("cpu_percent"),
memory_used=snapshot.get("memory_used"),
memory_free=snapshot.get("memory_free"),
net_in_bytes=snapshot.get("net_in_bytes"),
net_in_errors=snapshot.get("net_in_errors"),
net_in_dropped=snapshot.get("net_in_dropped"),
net_out_bytes=snapshot.get("net_out_bytes"),
net_out_errors= snapshot.get("net_out_errors"),
net_out_dropped=snapshot.get("net_out_dropped"),
insertion_date=collected_date,
insertion_time=collected_time)
def _save_disk_partition_snapshot(self, database, collected_date, collected_time):
for disk_partition in self.computer_snapshot.get("disk_snapshot_list"):
database.insertDiskPartitionSnapshot(
partition=disk_partition.partition,
used=disk_partition.disk_size_used,
free=disk_partition.disk_size_free,
mountpoint=';'.join(disk_partition.mountpoint_list),
insertion_date=collected_date,
insertion_time=collected_time)
slapos/collect/reporter.py 0 → 100644
View file @ 324177d5
#!/usr/bin/env python
from slapos.collect.db import Database
from slapos.util import mkdir_p
import os.path
import json
import csv
from time import strftime
class Dumper(object):
def __init__(self, database):
self.db = database
class SystemReporter(Dumper):
def dump(self, folder):
""" Dump data """
_date = strftime("%Y-%m-%d")
self.db.connect()
for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
self.writeFile(item, folder, collected_item_list)
for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems():
partition_id = partition.split("-")[0].split("/")[-1]
item = "memory_%s" % partition.split("-")[1]
self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)
self.db.close()
class SystemJSONReporterDumper(SystemReporter):
def writeFile(self, name, folder, collected_entry_list=[]):
""" Dump data as json """
file_io = open(os.path.join(folder, "system_%s.json" % name), "w")
json.dump(collected_entry_list, file_io, sort_keys=True, indent=2)
file_io.close()
class SystemCSVReporterDumper(SystemReporter):
def writeFile(self, name, folder, collected_entry_list=[]):
""" Dump data as json """
file_io = open(os.path.join(folder, "system_%s.csv" % name), "w")
csv_output = csv.writer(file_io)
csv_output.writerow(["time", "entry"])
for collected_entry in collected_entry_list:
csv_output.writerow([collected_entry["time"], collected_entry["entry"]])
file_io.close()
class RawDumper(Dumper):
""" Dump raw data in a certain format
"""
def dump(self, folder):
date = strftime("%Y-%m-%d")
self.db.connect()
table_list = self.db.getTableList()
for date_scope, amount in self.db.getDateScopeList(ignore_date=date):
for table in table_list:
self.writeFile(table, folder, date_scope,
self.db.select(table, date_scope))
self.db.markDayAsReported(date_scope,
table_list=table_list)
self.db.commit()
self.db.close()
class RawCSVDumper(RawDumper):
def writeFile(self, name, folder, date_scope, rows):
mkdir_p(os.path.join(folder, date_scope))
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
slapos/collect/snapshot.py 0 → 100644
View file @ 324177d5
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import psutil
import os
class _Snapshot(object):
def get(self, property, default=None):
return getattr(self, property, default)
class ProcessSnapshot(_Snapshot):
""" Take a snapshot from the running process
"""
def __init__(self, process=None):
assert type(process) is psutil.Process
ui_counter_list = process.get_io_counters()
self.username = process.username()
self.pid = process.pid
# Save full command line from the process.
self.name = "%s-%s" % (process.pid, process.create_time())
# CPU percentage, we will have to get actual absolute value
self.cpu_percent = process.get_cpu_percent(None)
# CPU Time
self.cpu_time = sum(process.get_cpu_times())
# Thread number, might not be really relevant
self.cpu_num_threads = process.get_num_threads()
# Memory percentage
self.memory_percent = process.get_memory_percent()
# Resident Set Size, virtual memory size is not accouned for
self.memory_rss = process.get_memory_info()[0]
# Byte count, Read and write. OSX NOT SUPPORTED
self.io_rw_counter = ui_counter_list[2] + ui_counter_list[3]
# Read + write IO cycles
self.io_cycles_counter = ui_counter_list[0] + ui_counter_list[1]
class SystemSnapshot(_Snapshot):
""" Take a snapshot from current system usage
"""
def __init__(self):
memory = psutil.phymem_usage()
net_io = psutil.net_io_counters()
self.memory_used = memory.used
self.memory_free = memory.free
self.memory_percent = memory.percent
self.cpu_percent = psutil.cpu_percent()
self.load = os.getloadavg()[0]
self.net_in_bytes = net_io.bytes_recv
self.net_in_errors = net_io.errin
self.net_in_dropped = net_io.dropin
self.net_out_bytes = net_io.bytes_sent
self.net_out_errors = net_io.errout
self.net_out_dropped = net_io.dropout
class DiskPartitionSnapshot(_Snapshot):
""" Take Snapshot from general disk partitions
usage
"""
def __init__(self, partition, mountpoint):
self.partition = partition
self.mountpoint_list = [ mountpoint ]
disk = psutil.disk_usage(mountpoint)
disk_io = psutil.disk_io_counters()
self.disk_size_used = disk.used
self.disk_size_free = disk.free
self.disk_size_percent = disk.percent
class ComputerSnapshot(_Snapshot):
""" Take a snapshot from computer informations
"""
def __init__(self):
self.cpu_num_core = psutil.NUM_CPUS
self.cpu_frequency = 0
self.cpu_type = 0
self.memory_size = psutil.TOTAL_PHYMEM
self.memory_type = 0
#
# Include a SystemSnapshot and a list DiskPartitionSnapshot
# on a Computer Snapshot
#
self.system_snapshot = SystemSnapshot()
self.disk_snapshot_list = []
self.partition_list = self._get_physical_disk_info()
def _get_physical_disk_info(self):
partition_dict = {}
for partition in psutil.disk_partitions():
if partition.device not in partition_dict:
usage = psutil.disk_usage(partition.mountpoint)
partition_dict[partition.device] = usage.total
self.disk_snapshot_list.append(
DiskPartitionSnapshot(partition.device,
partition.mountpoint))
return [(k, v) for k, v in partition_dict.iteritems()]
slapos/grid/slapgrid.py
View file @ 324177d5
...@@ -39,6 +39,7 @@ import tempfile ...@@ -39,6 +39,7 @@ import tempfile
import time import time
import traceback import traceback
import warnings import warnings
import logging
if sys.version_info < (2, 6): if sys.version_info < (2, 6):
warnings.warn('Used python version (%s) is old and has problems with' warnings.warn('Used python version (%s) is old and has problems with'
...@@ -48,6 +49,7 @@ from lxml import etree ...@@ -48,6 +49,7 @@ from lxml import etree
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.slap.slap import ServerError from slapos.slap.slap import ServerError
from slapos.util import mkdir_p, chownDirectory
from slapos.grid.exception import BuildoutFailedError from slapos.grid.exception import BuildoutFailedError
from slapos.grid.SlapObject import Software, Partition from slapos.grid.SlapObject import Software, Partition
from slapos.grid.svcbackend import launchSupervisord from slapos.grid.svcbackend import launchSupervisord
...@@ -606,6 +608,23 @@ class Slapgrid(object): ...@@ -606,6 +608,23 @@ class Slapgrid(object):
os.remove(timestamp_path) os.remove(timestamp_path)
self.logger.exception('') self.logger.exception('')
# Include Partition Logging
log_folder_path = "%s/.slapgrid/log" % instance_path
mkdir_p(log_folder_path)
partition_file_handler = logging.FileHandler(
filename="%s/instance.log" % (log_folder_path)
)
stat_info = os.stat(instance_path)
chownDirectory("%s/.slapgrid" % instance_path,
uid=stat_info.st_uid,
gid=stat_info.st_gid)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)-8s %(name)s %(message)s')
partition_file_handler.setFormatter(formatter)
self.logger.addHandler(partition_file_handler)
try:
self.logger.info('Processing Computer Partition %s.' % computer_partition_id) self.logger.info('Processing Computer Partition %s.' % computer_partition_id)
self.logger.info(' Software URL: %s' % software_url) self.logger.info(' Software URL: %s' % software_url)
self.logger.info(' Software path: %s' % software_path) self.logger.info(' Software path: %s' % software_path)
...@@ -626,7 +645,6 @@ class Slapgrid(object): ...@@ -626,7 +645,6 @@ class Slapgrid(object):
certificate_repository_path=self.certificate_repository_path, certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout, buildout=self.buildout,
logger=self.logger) logger=self.logger)
computer_partition_state = computer_partition.getState() computer_partition_state = computer_partition.getState()
# XXX this line breaks 37 tests # XXX this line breaks 37 tests
...@@ -663,6 +681,8 @@ class Slapgrid(object): ...@@ -663,6 +681,8 @@ class Slapgrid(object):
(computer_partition_id, computer_partition_state) (computer_partition_id, computer_partition_state)
computer_partition.error(error_string, logger=self.logger) computer_partition.error(error_string, logger=self.logger)
raise NotImplementedError(error_string) raise NotImplementedError(error_string)
finally:
self.logger.removeHandler(partition_file_handler)
# If partition has been successfully processed, write timestamp # If partition has been successfully processed, write timestamp
if timestamp: if timestamp:
...@@ -893,7 +913,7 @@ class Slapgrid(object): ...@@ -893,7 +913,7 @@ class Slapgrid(object):
try: try:
computer_partition_id = computer_partition.getId() computer_partition_id = computer_partition.getId()
#We want to execute all the script in the report folder # We want to execute all the script in the report folder
instance_path = os.path.join(self.instance_root, instance_path = os.path.join(self.instance_root,
computer_partition.getId()) computer_partition.getId())
report_path = os.path.join(instance_path, 'etc', 'report') report_path = os.path.join(instance_path, 'etc', 'report')
...@@ -902,7 +922,7 @@ class Slapgrid(object): ...@@ -902,7 +922,7 @@ class Slapgrid(object):
else: else:
script_list_to_run = [] script_list_to_run = []
#We now generate the pseudorandom name for the xml file # We now generate the pseudorandom name for the xml file
# and we add it in the invocation_list # and we add it in the invocation_list
f = tempfile.NamedTemporaryFile() f = tempfile.NamedTemporaryFile()
name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name)) name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
...@@ -914,13 +934,13 @@ class Slapgrid(object): ...@@ -914,13 +934,13 @@ class Slapgrid(object):
invocation_list = [] invocation_list = []
invocation_list.append(os.path.join(instance_path, 'etc', 'report', invocation_list.append(os.path.join(instance_path, 'etc', 'report',
script)) script))
#We add the xml_file name to the invocation_list # We add the xml_file name to the invocation_list
#f = tempfile.NamedTemporaryFile() #f = tempfile.NamedTemporaryFile()
#name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name)) #name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
#path_to_slapreport = os.path.join(instance_path, 'var', name_xml) #path_to_slapreport = os.path.join(instance_path, 'var', name_xml)
invocation_list.append(path_to_slapreport) invocation_list.append(path_to_slapreport)
#Dropping privileges # Dropping privileges
uid, gid = None, None uid, gid = None, None
stat_info = os.stat(instance_path) stat_info = os.stat(instance_path)
#stat sys call to get statistics informations #stat sys call to get statistics informations
...@@ -946,21 +966,25 @@ class Slapgrid(object): ...@@ -946,21 +966,25 @@ class Slapgrid(object):
self.logger.exception('Cannot run usage script(s) for %r:' % self.logger.exception('Cannot run usage script(s) for %r:' %
computer_partition.getId()) computer_partition.getId())
#Now we loop through the different computer partitions to report # Now we loop through the different computer partitions to report
report_usage_issue_cp_list = [] report_usage_issue_cp_list = []
for computer_partition in computer_partition_list: for computer_partition in computer_partition_list:
try: try:
filename_delete_list = [] filename_delete_list = []
computer_partition_id = computer_partition.getId() computer_partition_id = computer_partition.getId()
instance_path = os.path.join(self.instance_root, computer_partition_id) instance_path = os.path.join(self.instance_root, computer_partition_id)
dir_reports = os.path.join(instance_path, 'var', 'xml_report') dir_report_list = [os.path.join(instance_path, 'var', 'xml_report'),
#The directory xml_report contain a number of files equal os.path.join(self.instance_root, 'var', 'xml_report',
#to the number of software instance running inside the same partition computer_partition_id)]
for dir_reports in dir_report_list:
# The directory xml_report contain a number of files equal
# to the number of software instance running inside the same partition
if os.path.isdir(dir_reports): if os.path.isdir(dir_reports):
filename_list = os.listdir(dir_reports) filename_list = os.listdir(dir_reports)
else: else:
filename_list = [] filename_list = []
#self.logger.debug('name List %s' % filename_list) # self.logger.debug('name List %s' % filename_list)
for filename in filename_list: for filename in filename_list:
...@@ -968,7 +992,7 @@ class Slapgrid(object): ...@@ -968,7 +992,7 @@ class Slapgrid(object):
if os.path.exists(file_path): if os.path.exists(file_path):
usage = open(file_path, 'r').read() usage = open(file_path, 'r').read()
#We check the validity of xml content of each reports # We check the validity of xml content of each reports
if not self.validateXML(usage, partition_consumption_model): if not self.validateXML(usage, partition_consumption_model):
self.logger.info('WARNING: The XML file %s generated by slapreport is ' self.logger.info('WARNING: The XML file %s generated by slapreport is '
'not valid - This report is left as is at %s where you can ' 'not valid - This report is left as is at %s where you can '
...@@ -984,7 +1008,7 @@ class Slapgrid(object): ...@@ -984,7 +1008,7 @@ class Slapgrid(object):
else: else:
self.logger.debug('Usage report %r not found, ignored' % file_path) self.logger.debug('Usage report %r not found, ignored' % file_path)
#After sending the aggregated file we remove all the valid xml reports # After sending the aggregated file we remove all the valid xml reports
for filename in filename_delete_list: for filename in filename_delete_list:
os.remove(os.path.join(dir_reports, filename)) os.remove(os.path.join(dir_reports, filename))
...@@ -997,15 +1021,15 @@ class Slapgrid(object): ...@@ -997,15 +1021,15 @@ class Slapgrid(object):
self.logger.info('computer_partition_usage_list: %s - %s' % self.logger.info('computer_partition_usage_list: %s - %s' %
(computer_partition_usage.usage, computer_partition_usage.getId())) (computer_partition_usage.usage, computer_partition_usage.getId()))
#If there is, at least, one report # If there is, at least, one report
if computer_partition_usage_list != []: if computer_partition_usage_list != []:
try: try:
#We generate the final XML report with asXML method # We generate the final XML report with asXML method
computer_consumption = self.asXML(computer_partition_usage_list) computer_consumption = self.asXML(computer_partition_usage_list)
self.logger.info('Final xml report: %s' % computer_consumption) self.logger.info('Final xml report: %s' % computer_consumption)
#We test the XML report before sending it # We test the XML report before sending it
if self.validateXML(computer_consumption, computer_consumption_model): if self.validateXML(computer_consumption, computer_consumption_model):
self.logger.info('XML file generated by asXML is valid') self.logger.info('XML file generated by asXML is valid')
slap_computer_usage.reportUsage(computer_consumption) slap_computer_usage.reportUsage(computer_consumption)
......
slapos/tests/collect.py 0 → 100644
View file @ 324177d5
This diff is collapsed.
slapos/tests/slapgrid.py
View file @ 324177d5
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7