Commit f78c9d5c authored by Cédric Le Ninivin's avatar Cédric Le Ninivin

Introducing basic process manager for runner

parent 2a35d1f0
......@@ -8,6 +8,7 @@ import logging
import logging.handlers
from optparse import OptionParser, Option
import os
import slapos.runner.process
import sys
from slapos.runner.utils import runInstanceWithLock
......@@ -144,5 +145,6 @@ def serve(config):
if not os.path.exists(software_link):
os.mkdir(software_link)
runInstanceWithLock(app.config)
slapos.runner.process.setHandler()
app.run(host=config.runner_host, port=int(config.runner_port),
debug=config.debug, threaded=True)
import os
import psutil
import signal
import subprocess
import sys
SLAPRUNNER_PROCESS_LIST = []
class Popen(subprocess.Popen):
"""
Extension of Popen to launch and kill process in a clean way
"""
def __init__(self, *args, **kwargs):
"""
Launch process and add object to process list for handler
"""
self.name = kwargs.pop('name', None)
kwargs['stdin'] = subprocess.PIPE
kwargs['stderr'] = subprocess.STDOUT
kwargs.setdefault('stdout', subprocess.PIPE)
kwargs.setdefault('close_fds', True)
subprocess.Popen.__init__(self, *args, **kwargs)
SLAPRUNNER_PROCESS_LIST.append(self)
self.stdin.flush()
self.stdin.close()
self.stdin = None
def kill(self, sig=signal.SIGTERM, recursive=False):
"""
Kill process and all its descendant if recursive
"""
if self.poll() is None:
if recursive:
childs_pids = pidppid(self.pid)
for pid in childs_pids:
killNoFail(pid, sig)
killNoFail(self.pid, sig)
if self.poll() is not None:
SLAPRUNNER_PROCESS_LIST.remove(self)
def __del__(self):
"""
Del function, try to kill process group
and process if its group does not exist
"""
for pid in (-self.pid, self.pid):
try:
os.kill(-self.pid, 15)
except OSError:
pass
subprocess.Popen.__del__(self)
def pidppid(pid, recursive=True):
"""
Return a list of all children of pid
"""
return [p.pid for p in psutil.Process(pid).get_children(recursive=recursive)]
def killNoFail(pid, sig):
"""
function to kill without failing. Return True if kill do not fail
"""
try:
os.kill(pid, sig)
return True
except OSError:
return False
def isRunning(name):
"""
Return True if a process with this name is running
"""
for process in SLAPRUNNER_PROCESS_LIST:
if process.name == name:
if process.poll() is None:
return True
return False
def killRunningProcess(name, recursive=False):
"""
Kill all process with name
"""
for process in SLAPRUNNER_PROCESS_LIST:
if process.name == name:
process.kill(recursive=recursive)
def handler(sig, frame):
"""
Signal handler to kill all process
"""
pid = os.getpid()
os.kill(-pid, sig)
sys.exit()
def setHandler(sig_list=None):
if sig_list is None:
sig_list = [signal.SIGTERM]
for sig in sig_list:
signal.signal(sig, handler)
......@@ -83,10 +83,10 @@ function stopProcess() {
sendStop = true;
var urlfor = $SCRIPT_ROOT + "stopSlapgrid",
type = "slapgrid-sr.pid";
type = "slapgrid-sr";
if ($("#instrun").text() === "Stop") {
type = "slapgrid-cp.pid";
type = "slapgrid-cp";
}
$.post(urlfor, {type: type}, function (data) {
//if (data.result) {
......
......@@ -7,10 +7,9 @@ import md5
import logging
import multiprocessing
import re
import signal
from slapos.runner.process import Popen, isRunning, killRunningProcess
import shutil
import os
import subprocess
import time
import urllib
from xml.dom import minidom
......@@ -20,22 +19,12 @@ from flask import jsonify
import slapos.slap
# Setup default flask (werkzeug) parser
logger = logging.getLogger('werkzeug')
class Popen(subprocess.Popen):
def __init__(self, *args, **kwargs):
kwargs['stdin'] = subprocess.PIPE
kwargs['stderr'] = subprocess.STDOUT
kwargs.setdefault('stdout', subprocess.PIPE)
kwargs.setdefault('close_fds', True)
subprocess.Popen.__init__(self, *args, **kwargs)
self.stdin.flush()
self.stdin.close()
self.stdin = None
html_escape_table = {
"&": "&",
'"': """,
......@@ -177,54 +166,6 @@ def updateProxy(config):
computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
return True
def readPid(file):
"""Read process pid from file `file`"""
if os.path.exists(file):
data = open(file).read().strip()
try:
return int(data)
except Exception:
return 0
return 0
def writePid(file, pid):
"""Save process pid into a file `file`"""
open(file, 'w').write(str(pid))
def killRunningProcess(config, ptype):
"""Kill process and all running children process and remove pidfile"""
process_pid = os.path.join(config['run_dir'], ptype)
pid = readPid(process_pid)
if pid:
recursifKill([pid])
os.remove(process_pid)
else:
return False
def recursifKill(pids):
"""Try to kill a list of proccess by the given pid list"""
if pids == []:
return
else:
for pid in pids:
ppids = pidppid(pid)
try:
os.kill(pid, signal.SIGKILL) #kill current process
except Exception:
pass
recursifKill(ppids) #kill all children of this process
def pidppid(pid):
"""get the list of the children pids of a process `pid`"""
proc = Popen('ps -o pid,ppid ax | grep "%d"' % pid, shell=True,
stdout=subprocess.PIPE)
ppid = [x.split() for x in proc.communicate()[0].split("\n") if x]
return list(int(p) for p, pp in ppid if int(pp) == pid)
def updateInstanceParameter(config, software_type=None):
"""
Reconfigure Slapproxy to re-deploy current Software Instance with parameters.
......@@ -238,27 +179,17 @@ def updateInstanceParameter(config, software_type=None):
def startProxy(config):
"""Start Slapproxy server"""
proxy_pid = os.path.join(config['run_dir'], 'proxy.pid')
pid = readPid(proxy_pid)
running = False
if pid:
try:
os.kill(pid, 0)
except Exception:
pass
else:
running = True
if not running:
if not isRunning('slapproxy'):
log = os.path.join(config['log_dir'], 'slapproxy.log')
proxy = Popen([config['slapproxy'], '--log_file', log,
config['configuration_file_path']])
writePid(proxy_pid, proxy.pid)
Popen([config['slapproxy'], '--log_file', log,
config['configuration_file_path']],
name='slapproxy')
time.sleep(4)
def stopProxy(config):
"""Stop Slapproxy server"""
killRunningProcess(config,'proxy.pid')
pass
def removeProxyDb(config):
......@@ -267,22 +198,11 @@ def removeProxyDb(config):
if os.path.exists(config['database_uri']):
os.unlink(config['database_uri'])
def isSoftwareRunning(config):
def isSoftwareRunning(config=None):
"""
Return True if slapgrid-sr is still running and false if slapgrid if not
"""
slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
pid = readPid(slapgrid_pid)
if pid:
try:
os.kill(pid, 0)
except Exception:
running = False
else:
running = True
else:
running = False
return running
return isRunning('slapgrid-sr')
def runSoftwareWithLock(config):
......@@ -291,7 +211,7 @@ def runSoftwareWithLock(config):
compilation is done
"""
slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
if not isSoftwareRunning(config):
if not isSoftwareRunning():
if not os.path.exists(config['software_root']):
os.mkdir(config['software_root'])
stopProxy(config)
......@@ -304,9 +224,10 @@ def runSoftwareWithLock(config):
environment = os.environ.copy()
environment['MAKEFLAGS'] = '-j%r' % multiprocessing.cpu_count()
slapgrid = Popen([config['slapgrid_sr'], '-vc',
'--pidfile',slapgrid_pid,
'--pidfile', slapgrid_pid,
config['configuration_file_path'], '--now', '--develop'],
stdout=logfile, env=environment)
stdout=logfile, env=environment,
name='slapgrid-sr')
slapgrid.wait()
#Saves the current compile software for re-use
config_SR_folder(config)
......@@ -362,22 +283,11 @@ def loadSoftwareRList(config):
list.append(dict(md5=cfg[1], path=cfg[0], title=path))
return list
def isInstanceRunning(config):
def isInstanceRunning(config=None):
"""
Return True if slapgrid-cp is still running and false if slapgrid if not
"""
slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
pid = readPid(slapgrid_pid)
if pid:
try:
os.kill(pid, 0)
except Exception:
running = False
else:
running = True
else:
running = False
return running
return isRunning('slapgrid-cp')
def runInstanceWithLock(config):
......@@ -386,16 +296,16 @@ def runInstanceWithLock(config):
deployment is done.
"""
slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
if not isInstanceRunning(config):
if not isInstanceRunning():
startProxy(config)
logfile = open(config['instance_log'], 'w')
if not (updateProxy(config) and requestInstance(config)):
return False
svcStopAll(config) #prevent lost control of process
slapgrid = Popen([config['slapgrid_cp'], '-vc',
'--pidfile',slapgrid_pid,
'--pidfile', slapgrid_pid,
config['configuration_file_path'], '--now'],
stdout=logfile)
stdout=logfile, name='slapgrid-cp')
slapgrid.wait()
return True
return False
......@@ -587,10 +497,10 @@ def configNewSR(config, projectpath):
"""
folder = realpath(config, projectpath)
if folder:
if isInstanceRunning(config):
killRunningProcess(config, "slapgrid-cp.pid")
if isSoftwareRunning(config):
killRunningProcess(config, "slapgrid-sr.pid")
if isInstanceRunning():
killRunningProcess('slapgrid-cp')
if isSoftwareRunning():
killRunningProcess('slapgrid-sr')
stopProxy(config)
removeProxyDb(config)
startProxy(config)
......@@ -672,7 +582,7 @@ def removeSoftwareByName(config, md5, folderName):
config: slaprunner configuration
foldername: the link name given to the software release
md5: the md5 filename given by slapgrid to SR folder"""
if isSoftwareRunning(config) or isInstanceRunning(config):
if isSoftwareRunning() or isInstanceRunning():
raise Exception("Software installation or instantiation in progress, cannot remove")
path = os.path.join(config['software_root'], md5)
linkpath = os.path.join(config['software_link'], folderName)
......
......@@ -11,9 +11,10 @@ from flaskext.auth import Auth, AuthUser, login_required, logout
from flask import (Flask, request, redirect, url_for, render_template,
g, flash, jsonify, session, abort, send_file)
from slapos.runner.process import killRunningProcess
from slapos.runner.utils import (checkSoftwareFolder, configNewSR, getFolder, getFolderContent, getProfilePath,
getProjectList, getProjectTitle, getSession, getSlapStatus, getSvcStatus,
getSvcTailProcess, isInstanceRunning, isSoftwareRunning, isText, killRunningProcess,
getSvcTailProcess, isInstanceRunning, isSoftwareRunning, isText,
loadSoftwareRList, md5sum, newSoftware, readFileFrom, readParameters, realpath,
removeInstanceRoot, removeProxyDb, removeSoftwareByName, runInstanceWithLock,
runSoftwareWithLock, saveSession, svcStartStopProcess, svcStopAll, tail,
......@@ -427,7 +428,7 @@ def slapgridResult():
@login_required()
def stopSlapgrid():
result = killRunningProcess(app.config, request.form['type'])
result = killRunningProcess(request.form['type'])
return jsonify(result=result)
@login_required()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment