Merge branch 'slaprunner'

Conflicts:
	slapos/pubsub/notifier.py

setup.py

@@ -60,22 +60,22 @@ setup(name=name,
           'cloudmgr = slapos.cloudmgr.cloudmgr:main',
           'cloudstart = slapos.cloudmgr.start:main',
           'cloudstop = slapos.cloudmgr.stop:main',
-          'onetimeupload = slapos.onetimeupload:main',
+          'equeue = slapos.equeue:main',
+          'killpidfromfile = slapos.systool:killpidfromfile',
+          'lampconfigure = slapos.lamp:run [lampconfigure]',
           'onetimedownload = slapos.onetimedownload:main',
+          'onetimeupload = slapos.onetimeupload:main',
+          'pubsubnotifier = slapos.pubsub.notifier:main',
+          'pubsubserver = slapos.pubsub:main',
           'shacache = slapos.shacache:main',
           'slapbuilder = slapos.builder:main',
+          'slapcontainer = slapos.container:main',
           'slapmonitor = slapos.monitor:run_slapmonitor',
           'slapmonitor-xml = slapos.monitor:run_slapmonitor_xml',
           'slapreport = slapos.monitor:run_slapreport',
           'slaprunner = slapos.runner:run',
-          'killpidfromfile = slapos.systool:killpidfromfile',
-          'lampconfigure = slapos.lamp:run [lampconfigure]',
-          'equeue = slapos.equeue:main',
-          'pubsubserver = slapos.pubsub:main',
-          'pubsubnotifier = slapos.pubsub.notifier:main',
           'slaprunnertest = slapos.runner.runnertest:main',
           'zodbpack = slapos.zodbpack:run [zodbpack]',
-          'slapcontainer = slapos.container:main',
         ]
       },
     )

slapos/builder/__init__.py

@@ -152,19 +152,19 @@ def run(config):
     fdisk_output_path = "/tmp/a_generated_file"
   try:
     if not config.virtual:
-    	offset = 0
-    	fdisk_output_file = open(fdisk_output_path, 'w')
-    	_call(['sfdisk', '-d', '-uS', config.system_path], stdout=fdisk_output_file)
-    	fdisk_output_file.close()
-    	fdisk_output_file = open(fdisk_output_path, 'r')
-    	for line in fdisk_output_file:
-    	  line = line.rstrip().replace(' ', '')
-    	  if line.endswith("bootable"):
-    	    offset = int(line.split(':')[1].split(',')[0].split('=')[1])
-    	fdisk_output_file.close()
-    	offset = offset * 512
-    	_call(['mount', '-o', 'loop,offset=%i' % offset, config.system_path,
-    	       mount_dir_path], dry_run=dry_run)
+      offset = 0
+      fdisk_output_file = open(fdisk_output_path, 'w')
+      _call(['sfdisk', '-d', '-uS', config.system_path], stdout=fdisk_output_file)
+      fdisk_output_file.close()
+      fdisk_output_file = open(fdisk_output_path, 'r')
+      for line in fdisk_output_file:
+        line = line.rstrip().replace(' ', '')
+        if line.endswith("bootable"):
+          offset = int(line.split(':')[1].split(',')[0].split('=')[1])
+      fdisk_output_file.close()
+      offset = offset * 512
+      _call(['mount', '-o', 'loop,offset=%i' % offset, config.system_path,
+             mount_dir_path], dry_run=dry_run)
     # Call vmware-mount to mount Virtual disk image
     else:
       print "Mount Virtual Image"
@@ -191,7 +191,7 @@ def run(config):
                                               cert_file]))
       for (src, dst) in [(config.key_file, key_file_dest), (config.cert_file,
         cert_file_dest)]:
-        print "Coping %r to %r, and setting minimum privileges" % (src, dst)
+        print "Coping %r to %r, and setting minimal privileges" % (src, dst)
         if not dry_run:
           shutil.copy(src, dst)
           os.chmod(dst, 0600)
@@ -302,16 +302,16 @@ def run(config):
         
       # Adding slapos_firstboot in case of MultiDisk usage    
       if not config.one_disk :
-      	for script in ['slapos_firstboot']:
-      	  path = os.path.join(mount_dir_path, 'etc', 'init.d', script)
-      	  print "Creating %r" % path
-      	  if not dry_run:
-      	    open(path, 'w').write(pkg_resources.resource_stream(__name__,
-      	      'script/%s' % script).read())
-      	    os.chmod(path, 0755)	  
+        for script in ['slapos_firstboot']:
+          path = os.path.join(mount_dir_path, 'etc', 'init.d', script)
+          print "Creating %r" % path
+          if not dry_run:
+            open(path, 'w').write(pkg_resources.resource_stream(__name__,
+              'script/%s' % script).read())
+            os.chmod(path, 0755)
       else:
         for script in ['slapos_firstboot']:
-      	  path = os.path.join(mount_dir_path, 'etc', 'init.d', script)
+          path = os.path.join(mount_dir_path, 'etc', 'init.d', script)
           if os.path.exists(path):
             print "Removing %r" % path
             os.remove(path)

slapos/runner/__init__.py

-from optparse import OptionParser, Option
+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+# pylint: disable-msg=W0311,C0301,C0103,C0111,R0904,R0903
+
 import ConfigParser
+import datetime
 import logging
 import logging.handlers
+from optparse import OptionParser, Option
 import os
+import slapos.runner.process
 import sys
-import subprocess
-from datetime import timedelta
+from slapos.runner.utils import runInstanceWithLock
+
 
 class Parser(OptionParser):
   """
@@ -13,7 +19,7 @@ class Parser(OptionParser):
   """
   def __init__(self, usage=None, version=None):
     """
-    Initialize all options possibles.
+    Initialize all possible options.
     """
     OptionParser.__init__(self, usage=usage, version=version,
                           option_list=[
@@ -45,6 +51,13 @@ class Parser(OptionParser):
     return options, args[0]
 
 class Config:
+  def __init__(self):
+    self.configuration_file_path = None
+    self.console = None
+    self.log_file = None
+    self.logger = None
+    self.verbose = None
+
   def setConfig(self, option_dict, configuration_file_path):
     """
     Set options given by parameters.
@@ -100,9 +113,13 @@ def run():
     config = Config()
     config.setConfig(*Parser(usage=usage).check_args())
 
+    if os.getuid() == 0:
+        # avoid mistakes (mainly in development mode)
+        raise Exception('Do not run SlapRunner as root.')
+
     serve(config)
     return_code = 0
-  except SystemExit, err:
+  except SystemExit as err:
     # Catch exception raise by optparse
     return_code = err
 
@@ -121,11 +138,13 @@ def serve(config):
     instance_profile='instance.cfg',
     software_profile='software.cfg',
     SECRET_KEY=os.urandom(24),
-    PERMANENT_SESSION_LIFETIME=timedelta(days=31),
+    PERMANENT_SESSION_LIFETIME=datetime.timedelta(days=31),
   )
   if not os.path.exists(workdir):
     os.mkdir(workdir)
   if not os.path.exists(software_link):
     os.mkdir(software_link)
+  slapos.runner.process.setHandler()
+  runInstanceWithLock(app.config)
   app.run(host=config.runner_host, port=int(config.runner_port),
       debug=config.debug, threaded=True)

slapos/runner/decorators.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
+#pylint: disable-all
+
+import json
+
+from flask import Response
+
+
+def as_json(f):
+  def inner(*args, **kwargs):
+    return Response(json.dumps(f(*args, **kwargs)), mimetype='application/json')
+  return inner
+

slapos/runner/gittools.py

 # -*- coding: utf-8 -*-
+# vim: set et sts=2:
+# pylint: disable-msg=W0311,C0301,C0103,C0111
+
 
-import slapos.slap
-import time
-import subprocess
 import os
 import re
-import urllib
-from flask import jsonify
 import shutil
-import string
-from git import *
 
-class Popen(subprocess.Popen):
-  def __init__(self, *args, **kwargs):
-    kwargs['stdin'] = subprocess.PIPE
-    kwargs['stderr'] = subprocess.STDOUT
-    kwargs.setdefault('stdout', subprocess.PIPE)
-    kwargs.setdefault('close_fds', True)
-    subprocess.Popen.__init__(self, *args, **kwargs)
-    self.stdin.flush()
-    self.stdin.close()
-    self.stdin = None
+from git import Repo
+from flask import jsonify
+
 
 def cloneRepo(data):
   """Clonne a repository
@@ -49,7 +38,7 @@ def cloneRepo(data):
     if data["email"] != "":
       config_writer.set_value("user", "email", data["email"])
     code = 1
-  except Exception, e:
+  except Exception as e:
     json = safeResult(str(e))
     if os.path.exists(workDir):
       shutil.rmtree(workDir)
@@ -70,7 +59,7 @@ def gitStatus(project):
     branch = git.branch().replace(' ', '').split('\n')
     isdirty = repo.is_dirty(untracked_files=True)
     code = 1
-  except Exception, e:
+  except Exception as e:
     json = safeResult(str(e))
   return jsonify(code=code, result=json, branch=branch, dirty=isdirty)
 
@@ -85,7 +74,6 @@ def switchBranch(project, name):
   json = ""
   try:
     repo = Repo(project)
-    branches = repo.branches
     current_branch = repo.active_branch.name
     if name == current_branch:
       json = "This is already your active branch for this project"
@@ -93,7 +81,7 @@ def switchBranch(project, name):
       git  = repo.git
       git.checkout(name)
       code = 1
-  except Exception, e:
+  except Exception as e:
     json = safeResult(str(e))
   return jsonify(code=code, result=json)
 
@@ -115,7 +103,7 @@ def addBranch(project, name, onlyCheckout=False):
     else:
       git.checkout(name)
     code = 1
-  except Exception, e:
+  except Exception as e:
     json = safeResult(str(e))
   return jsonify(code=code, result=json)
 
@@ -127,7 +115,7 @@ def getDiff(project):
     git = repo.git
     current_branch = repo.active_branch.name
     result = git.diff(current_branch)
-  except Exception, e:
+  except Exception as e:
     result = safeResult(str(e))
   return result
 
@@ -157,7 +145,7 @@ def gitPush(project, msg):
     else:
       json = "Nothing to be commited"
       code = 1
-  except Exception, e:
+  except Exception as e:
     if undo_commit:
       git.reset("HEAD~") #undo previous commit
     json = safeResult(str(e))
@@ -169,14 +157,13 @@ def gitPull(project):
   try:
     repo = Repo(project)
     git = repo.git
-    current_branch = repo.active_branch.name
     git.pull()
     code = 1
-  except Exception, e:
+  except Exception as e:
     result = safeResult(str(e))
   return jsonify(code=code, result=result)
 
 def safeResult(result):
   """Parse string and remove credential of the user"""
-  regex=re.compile("(https:\/\/)([\w\d\._-]+:[\w\d\._-]+)\@([\S]+\s)", re.VERBOSE)
-  return regex.sub(r'\1\3', result)
\ No newline at end of file
+  regex = re.compile("(https:\/\/)([\w\d\._-]+:[\w\d\._-]+)\@([\S]+\s)", re.VERBOSE)
+  return regex.sub(r'\1\3', result)

slapos/runner/process.py

+import os
+import psutil
+import signal
+import subprocess
+import sys
+
+SLAPRUNNER_PROCESS_LIST = []
+
+
+class Popen(subprocess.Popen):
+  """
+  Extension of Popen to launch and kill process in a clean way
+  """
+  def __init__(self, *args, **kwargs):
+    """
+    Launch process and add object to process list for handler
+    """
+    self.name = kwargs.pop('name', None)
+    kwargs['stdin'] = subprocess.PIPE
+    kwargs['stderr'] = subprocess.STDOUT
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    kwargs.setdefault('close_fds', True)
+    subprocess.Popen.__init__(self, *args, **kwargs)
+    SLAPRUNNER_PROCESS_LIST.append(self)
+    self.stdin.flush()
+    self.stdin.close()
+    self.stdin = None
+
+  def kill(self, sig=signal.SIGTERM, recursive=False):
+    """
+    Kill process and all its descendant if recursive
+    """
+    if self.poll() is None:
+      if recursive:
+        childs_pids = pidppid(self.pid)
+        for pid in childs_pids:
+          killNoFail(pid, sig)
+      killNoFail(self.pid, sig)
+      if self.poll() is not None:
+        SLAPRUNNER_PROCESS_LIST.remove(self)
+
+  def __del__(self):
+    """
+    Del function, try to kill process group
+    and process if its group does not exist
+    """
+    for pid in (-self.pid, self.pid):
+      try:
+        os.kill(-self.pid, 15)
+      except OSError:
+        pass
+    subprocess.Popen.__del__(self)
+
+
+def pidppid(pid, recursive=True):
+  """
+  Return a list of all children of pid
+  """
+  return [p.pid for p in psutil.Process(pid).get_children(recursive=recursive)]
+
+
+def killNoFail(pid, sig):
+  """
+  function to kill without failing. Return True if kill do not fail
+  """
+  try:
+    os.kill(pid, sig)
+    return True
+  except OSError:
+    return False
+
+
+def isRunning(name):
+  """
+  Return True if a process with this name is running
+  """
+  for process in SLAPRUNNER_PROCESS_LIST:
+    if process.name == name:
+      if process.poll() is None:
+        return True
+  return False
+
+
+def killRunningProcess(name, recursive=False):
+  """
+  Kill all process with name
+  """
+  for process in SLAPRUNNER_PROCESS_LIST:
+    if process.name == name:
+      process.kill(recursive=recursive)
+
+
+def handler(sig, frame):
+  """
+  Signal handler to kill all process
+  """
+  pid = os.getpid()
+  os.kill(-pid, sig)
+  sys.exit()
+
+
+def setHandler(sig_list=None):
+  if sig_list is None:
+    sig_list = [signal.SIGTERM]
+  for sig in sig_list:
+    signal.signal(sig, handler)

slapos/runner/static/css/gsFileManager.css

 UL.jqueryFileTree {
-	font-family: Verdana, sans-serif;
+  font-family: Verdana, sans-serif;
 	font-size: 12px;
 	line-height: 18px;
 	padding: 0px;
@@ -60,6 +60,7 @@ UL.jqueryFileTree A:hover {
     background-color: white;
     border: 1px solid #BBBBBB;
 }
+div.browser div.gs_dir_content_files{min-height:330px;}
 
 #curDir {
  height: 30px;
@@ -124,7 +125,7 @@ UL.jqueryFileTree A:hover {
     font-weight: bold;
 }
 .gs_head{padding: 10px 5px 10px 0;}
-.gs_title{padding-left: 10px; margin:5px 5px 0 0; background: #e4e4e4;}
+.gs_title{padding-left: 10px; margin:5px 5px 0 0; background: #e4e4e4; height:30px;}
 .rowSelected {
 	color: #FFF;
 	background-color: #c2c2c2;
@@ -133,6 +134,7 @@ UL.jqueryFileTree A:hover {
 .rowSelected a {
 	color: #FFF;
 }
+.pathlink {cursor: pointer; font-size: 13px; margin-right: 2px; margin-left: 2px;}
 
 .gs_delimiter {
     height: 16px;

slapos/runner/static/css/jqueryTabs.css

@@ -2,7 +2,7 @@
 	background: #e4e4e4;
 	padding:3px;
 	/*position:relative;*/
-	width:757px;
+	width:930px;
 	font-size: 14px;
   border-radius: 4px 4px 0 0;
 }
@@ -10,9 +10,9 @@
 #tabContaier textarea {
 	width:702px;
 }
-#tabContaier textarea.slap{white-space: pre-wrap;word-wrap: break-word;overflow: hidden;color: #6F6F6F;width:430px; max-height:120px;
+#tabContaier textarea.slap{white-space: pre-wrap;word-wrap: break-word;overflow: hidden;color: #6F6F6F;width:616px; max-height:120px;
   resize: none; height:18px;padding:3px;min-height:18px;font-size: 13px;}
-#tabContaier textarea.mb_style{width:560px;}
+#tabContaier textarea.mb_style{width:713px;}
 #tabContaier > ul{
 	overflow:hidden;
 	height:34px;

slapos/runner/static/css/styles.css

@@ -28,7 +28,7 @@ td{
     border-right: 1px solid #A8A8A8;
 }
 td.propertie{padding-left:10px; color:blue;width: 178px;}
-.slapvalues{display:block;width: 375px;word-wrap: break-word}
+.slapvalues{display:block;width: 522px;word-wrap: break-word}
 td.first{border-left: 1px solid #A8A8A8;}
 th{
     background: #2281C1;
@@ -39,7 +39,7 @@ th{
 }
 table.small th{padding: 4px;font-size: 16px;}
 textarea {
-	width:762px;
+	width:932px;
   font-family: 'Helvetica Neue',Tahoma,Helvetica,Arial,sans-serif;
 }
 
@@ -54,8 +54,8 @@ overflow-y: scroll;
 
 #page
 {
-	margin: 20px auto;
-	width: 841px;
+	margin: 10px auto;
+	width: 1014px;
 }
 
 #logo{
@@ -64,8 +64,8 @@ overflow-y: scroll;
 
 #header{
     height: 78px;
-    padding-top: 15px;
-    width: 847px;
+    padding-top: 12px;
+    width: 1014px;
     background: url(../images/head.png) no-repeat;
 }
 
@@ -86,7 +86,7 @@ overflow-y: scroll;
     font-weight: normal;
     padding-top: 3px;
     float: left;
-    width: 656px;
+    width: 820px;
     height: 22px;
     text-align: center;
     color: #4c6172;
@@ -114,16 +114,12 @@ overflow-y: scroll;
     }
 
 .wmenu{
-
+  margin: 1px 0 1px 20px;
 }
 
 .wmenu ul{
-    list-style:none;
-    margin:0;
-    margin-top:1px;
-    padding:0;
-    margin-left:20px;
-    height: 36px;
+    height: 34px;
+    overflow:hidden;
 }
 
 .space{
@@ -132,7 +128,6 @@ overflow-y: scroll;
 
 .wmenu li{
     float:left;
-    background: url(../images/sep.png) left center no-repeat;
 }
 
 .wmenu li .title_software{
@@ -146,7 +141,7 @@ overflow-y: scroll;
     padding-left: 20px;
     padding-right: 5px;
     margin-left:4px;
-    height: 20px;
+    height: 18px;
     text-shadow:1px 1px 0 #E6EDF2;
 }
 
@@ -165,20 +160,23 @@ overflow-y: scroll;
     text-shadow:1px 1px 0 #E6EDF2;
 }
 
-
+.wmenu ul li.sep{border-right: 1px solid #c2c2c2; min-width:20px; height:34px}
 .wmenu ul li a{
     display:block;
     height: 20px;
-    Color:#000;
-    font-size:14px;
+    Color:#275777;
+    /*background: url(../images/sep.png) right center no-repeat;*/
+    border-right: 1px solid #c2c2c2;
+    /*font-weight:bold;*/
+    font-size:15px;
     text-decoration:none;
-    padding:8px;
-    padding-left: 6px;
+    padding:7px 10px 7px 10px;
 }
+.wmenu ul li:last-child a {border: none}
 
 .wmenu ul li a:hover, .wmenu ul li a.active{
-    color:#0271BF;
-    background:url(../images/menu_hover.png) bottom repeat-x;
+    color: #000;/*#0271BF;*/
+    background:#c2c2c2;
 }
 
 #main
@@ -191,7 +189,7 @@ overflow-y: scroll;
 
 .main_content{
     position:relative;
-    width: 100%;
+    width: 99.6%;
     height: 430px;
     margin-top:10px;
     border:1px solid #4c6172;
@@ -201,12 +199,12 @@ overflow-y: scroll;
 .main_head{
     height: 15px;
     background: url(../images/main_top.png) no-repeat;
-    width: 794px;
+    width: 966px;
 }
 
 .content{
     background: url(../images/content.png) repeat-y;
-    width: 764px;
+    width: 936px;
     padding: 15px;
     padding-top:0;
     padding-bottom:0;
@@ -229,16 +227,16 @@ overflow-y: scroll;
 .main_foot{
     height: 15px;
     background: url(../images/main_bottom.png) no-repeat;
-    width: 794px;
+    width: 966px;
 }
 
 #footer{
     height: 35px;
-    width: 767px;
+    width: 934px;
     background: url(../images/footer_bg.png) no-repeat;
     padding-left: 40px;
     padding-right: 40px;
-    padding-top: 5px;
+    padding-top: 4px;
     text-align: center;
     color: #4c6172;
     text-shadow: 0px 1px 1px #fff;
@@ -319,7 +317,8 @@ input[type="radio"], input[type="checkbox"]{
     border: 2px solid #87B0D4;
     padding: 0;
     color: #4c6172;
-    margin: 15px 59px 15px 59px;
+    margin: 20px auto;
+    width: 641px;
 }
 
 #home_box a{
@@ -369,14 +368,14 @@ input[type="radio"], input[type="checkbox"]{
 .lmenu{
     display: block;
     height: 80px;
-    margin: 10px 45px 10px 45px;
+    margin: 10px 0 10px 45px;
     padding: 10px;
     padding-bottom:15px;
     width: 530px;
     border: 1px solid #678dad;
 }
 
-.smaller{margin-left:10px; width: 350px; float:right}
+.smaller{margin-left:10px; width: 350px; float:left}
 .smaller p{width: 280px;}
 .umenu{display: block;margin: 10px 0 10px 45px; width: 147px; float:left;border: 1px solid #678dad;
 padding: 10px;height: 80px;padding-bottom:15px;}
@@ -398,7 +397,6 @@ padding: 10px;height: 80px;padding-bottom:15px;}
 .sright_menu{
     display: block;
     height: 80px;
-    margin: 0 45px 0 0;
     padding: 10px;
     padding-bottom:15px;
     border: #678dad;
@@ -415,7 +413,7 @@ padding: 10px;height: 80px;padding-bottom:15px;}
 }
 
 .file_tree {
-    width: 752px;
+    width: 923px;
     height: 300px;
     border: solid 1px #678dad;
     background: #fff;
@@ -433,28 +431,30 @@ padding: 10px;height: 80px;padding-bottom:15px;}
 }
 
 .file_tree_short{
-    width: 352px;
-    height: 170px;
+    width: 228px;
+    height: 379px;
     border: solid 1px #678dad;
     background: #fff;
     overflow: auto;
     padding: 5px;
-    float:left;
 }
-
 .box_software{
-    width: 362px;
-    height: 100px;
+    width: 240px;
     background: #fff;
-    padding: 5px;
-    float:right;
 }
+.software_details{float: left;}
 #details_box{
-    display:none;
+    display:block;
+    margin-top:10px;
 }
+#code{
+    float: right;
+    width: 692px;
+}
+#details_head{margin-bottom: 10px;}
 
 #contentInfo{
-    width: 756px;
+    width: 926px;
     border: solid 1px #678dad;
     background: #fff;
     overflow: auto;
@@ -490,10 +490,6 @@ h2.hight:hover{
     cursor: pointer;
 }
 
-.software_details{
-    padding: 10px;