Commit 3bb36903 authored by Xavier Thompson's avatar Xavier Thompson

slapgrid: Process promises with SR python

See merge request nexedi/slapos.core!329
parents b1dd96b1 45320b33
...@@ -45,7 +45,8 @@ from six.moves.configparser import ConfigParser ...@@ -45,7 +45,8 @@ from six.moves.configparser import ConfigParser
from supervisor import xmlrpc from supervisor import xmlrpc
from slapos.grid.utils import (md5digest, getCleanEnvironment, from slapos.grid.utils import (md5digest, getCleanEnvironment,
SlapPopen, dropPrivileges, updateFile) SlapPopen, dropPrivileges, updateFile,
getPythonExecutableFromSoftwarePath)
from slapos.grid import utils # for methods that could be mocked, access them through the module from slapos.grid import utils # for methods that could be mocked, access them through the module
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.grid.svcbackend import getSupervisorRPC from slapos.grid.svcbackend import getSupervisorRPC
...@@ -472,6 +473,8 @@ class Partition(object): ...@@ -472,6 +473,8 @@ class Partition(object):
self.instance_min_free_space = instance_min_free_space self.instance_min_free_space = instance_min_free_space
self.instance_python = getPythonExecutableFromSoftwarePath(self.software_path)
def check_free_space(self): def check_free_space(self):
required = self.instance_min_free_space or 0 required = self.instance_min_free_space or 0
...@@ -705,6 +708,7 @@ class Partition(object): ...@@ -705,6 +708,7 @@ class Partition(object):
debug=self.buildout_debug) debug=self.buildout_debug)
self.generateSupervisorConfigurationFile() self.generateSupervisorConfigurationFile()
self.createRetentionLockDelay() self.createRetentionLockDelay()
self.instance_python = getPythonExecutableFromSoftwarePath(self.software_path)
def generateSupervisorConfiguration(self): def generateSupervisorConfiguration(self):
""" """
......
...@@ -44,7 +44,7 @@ import hashlib ...@@ -44,7 +44,7 @@ import hashlib
from datetime import datetime from datetime import datetime
from multiprocessing import Process, Queue as MQueue from multiprocessing import Process, Queue as MQueue
from six.moves import queue, reload_module from six.moves import queue, reload_module
from slapos.util import str2bytes, mkdir_p, chownDirectory from slapos.util import str2bytes, mkdir_p, chownDirectory, listifdir
from slapos.grid.utils import dropPrivileges, killProcessTree from slapos.grid.utils import dropPrivileges, killProcessTree
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult, from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
...@@ -731,10 +731,8 @@ class PromiseLauncher(object): ...@@ -731,10 +731,8 @@ class PromiseLauncher(object):
error = 0 error = 0
success = 0 success = 0
promise_name_list = [] promise_name_list = []
if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder): for promise_name in listifdir(self.promise_folder):
for promise_name in os.listdir(self.promise_folder): if promise_name.endswith(('.pyc', '.pyo')):
for suffix in ['.pyc', '.pyo']:
if promise_name.endswith(suffix):
promise_path = os.path.join(self.promise_folder, promise_name) promise_path = os.path.join(self.promise_folder, promise_name)
if not os.path.exists(promise_path[:-1]): if not os.path.exists(promise_path[:-1]):
try: try:
...@@ -816,10 +814,9 @@ class PromiseLauncher(object): ...@@ -816,10 +814,9 @@ class PromiseLauncher(object):
if key not in promise_name_list: if key not in promise_name_list:
new_state_dict.pop(key, None) new_state_dict.pop(key, None)
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \ if not self.run_only_promise_list:
and os.path.isdir(self.legacy_promise_folder):
# run legacy promise styles # run legacy promise styles
for promise_name in os.listdir(self.legacy_promise_folder): for promise_name in listifdir(self.legacy_promise_folder):
promise_path = os.path.join(self.legacy_promise_folder, promise_name) promise_path = os.path.join(self.legacy_promise_folder, promise_name)
if not os.path.isfile(promise_path) or \ if not os.path.isfile(promise_path) or \
not os.access(promise_path, os.X_OK): not os.access(promise_path, os.X_OK):
......
from __future__ import print_function
import argparse
import ast
import os
import sys
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('--promise-folder', required=True)
parser.add_argument('--legacy-promise-folder', default=None)
parser.add_argument('--promise-timeout', type=int, default=20)
parser.add_argument('--partition-folder', default=None)
parser.add_argument('--log-folder', default=None)
parser.add_argument('--force', action='store_true')
parser.add_argument('--check-anomaly', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--master-url', default=None)
parser.add_argument('--partition-cert', default=None)
parser.add_argument('--partition-key', default=None)
parser.add_argument('--partition-id', default=None)
parser.add_argument('--computer-id', default=None)
args = parser.parse_args()
# Extract slapos.core path and all dependencies from first promise found
# to import slapos.core
promise_folder = args.promise_folder
promise_file = next(
p for p in os.listdir(promise_folder)
if p.endswith('.py') and not p.startswith('__init__')
)
with open(os.path.join(promise_folder, promise_file)) as f:
promise_content = f.read()
tree = ast.parse(promise_content, mode='exec')
sys.path[0:0] = eval(compile(ast.Expression(tree.body[1].value), '', 'eval'))
from slapos.grid.promise import PromiseLauncher, PromiseError
from slapos.cli.entry import SlapOSApp
# Configure promise launcher
# with the same logger as standard slapos command
app = SlapOSApp()
app.options, _ = app.parser.parse_known_args([])
app.configure_logging()
config = {k.replace('_', '-') : v for k, v in vars(args).items()}
promise_checker = PromiseLauncher(config=config, logger=app.log)
# Run promises
# Redirect stdout to stderr (logger only uses stderr already)
# to reserve stdout for error reporting
out = os.dup(1)
os.dup2(2, 1)
try:
promise_checker.run()
except Exception as e:
os.write(out, str(e))
sys.exit(2 if isinstance(e, PromiseError) else 1)
This diff is collapsed.
...@@ -164,6 +164,20 @@ def md5digest(url): ...@@ -164,6 +164,20 @@ def md5digest(url):
return hashlib.md5(url.encode('utf-8')).hexdigest() return hashlib.md5(url.encode('utf-8')).hexdigest()
def getPythonExecutableFromSoftwarePath(software_path):
"""
Return the path of the python executable installed for the software release
installed as `software_path`.
"""
try:
with open(os.path.join(software_path, 'bin', 'buildout')) as f:
shebang = f.readline()
except OSError:
return
if shebang.startswith('#!'):
return shebang[2:].split(None, 1)[0]
def getCleanEnvironment(logger, home_path='/tmp'): def getCleanEnvironment(logger, home_path='/tmp'):
changed_env = {} changed_env = {}
removed_env = [] removed_env = []
......
...@@ -41,6 +41,8 @@ from contextlib import closing ...@@ -41,6 +41,8 @@ from contextlib import closing
from six.moves import BaseHTTPServer from six.moves import BaseHTTPServer
from six.moves import urllib_parse from six.moves import urllib_parse
from ..grid.utils import getPythonExecutableFromSoftwarePath
try: try:
import typing import typing
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
...@@ -77,8 +79,12 @@ def getPromisePluginParameterDict(filepath): ...@@ -77,8 +79,12 @@ def getPromisePluginParameterDict(filepath):
This allow to check that monitoring plugin are using a proper config. This allow to check that monitoring plugin are using a proper config.
""" """
executable = getPythonExecutableFromSoftwarePath(
os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(filepath))),
'software_release'))
extra_config_dict_json = subprocess.check_output([ extra_config_dict_json = subprocess.check_output([
sys.executable, executable,
"-c", "-c",
""" """
import json, sys import json, sys
......
...@@ -45,6 +45,7 @@ import json ...@@ -45,6 +45,7 @@ import json
import re import re
import grp import grp
import hashlib import hashlib
import errno
import mock import mock
from mock import patch from mock import patch
...@@ -65,6 +66,11 @@ import slapos.grid.SlapObject ...@@ -65,6 +66,11 @@ import slapos.grid.SlapObject
from slapos import manager as slapmanager from slapos import manager as slapmanager
from slapos.util import dumps from slapos.util import dumps
from slapos import __path__ as slapos_path
from zope import __path__ as zope_path
PROMISE_PATHS = sorted(set(map(os.path.dirname, slapos_path + zope_path)))
import httmock import httmock
...@@ -113,6 +119,9 @@ touch worked ...@@ -113,6 +119,9 @@ touch worked
""" """
PROMISE_CONTENT_TEMPLATE = """ PROMISE_CONTENT_TEMPLATE = """
import sys
sys.path[0:0] = %(paths)r
from zope.interface import implementer from zope.interface import implementer
from slapos.grid.promise import interface from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise from slapos.grid.promise import GenericPromise
...@@ -122,7 +131,7 @@ class RunPromise(GenericPromise): ...@@ -122,7 +131,7 @@ class RunPromise(GenericPromise):
def __init__(self, config): def __init__(self, config):
super(RunPromise, self).__init__(config) super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=%(periodicity)s) self.setPeriodicity(minute=%(periodicity)r)
def sense(self): def sense(self):
%(content)s %(content)s
...@@ -133,10 +142,10 @@ class RunPromise(GenericPromise): ...@@ -133,10 +142,10 @@ class RunPromise(GenericPromise):
self.logger.info("success") self.logger.info("success")
def anomaly(self): def anomaly(self):
return self._anomaly(result_count=2, failure_amount=%(failure_amount)s) return self._anomaly(result_count=2, failure_amount=%(failure_amount)r)
def test(self): def test(self):
return self._test(result_count=1, failure_amount=%(failure_amount)s) return self._test(result_count=1, failure_amount=%(failure_amount)r)
""" """
class BasicMixin(object): class BasicMixin(object):
...@@ -150,6 +159,14 @@ class BasicMixin(object): ...@@ -150,6 +159,14 @@ class BasicMixin(object):
del os.environ['SLAPGRID_INSTANCE_ROOT'] del os.environ['SLAPGRID_INSTANCE_ROOT']
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.setSlapgrid() self.setSlapgrid()
self.setMock()
def setMock(self):
module = slapos.grid.SlapObject
func = 'getPythonExecutableFromSoftwarePath'
orig = getattr(module, func)
self.addCleanup(setattr, module, func, orig)
setattr(module, func, lambda software_path: None)
def setSlapgrid(self, develop=False, force_stop=False): def setSlapgrid(self, develop=False, force_stop=False):
if getattr(self, 'master_url', None) is None: if getattr(self, 'master_url', None) is None:
...@@ -576,7 +593,8 @@ class InstanceForTest(object): ...@@ -576,7 +593,8 @@ class InstanceForTest(object):
{'success': success, {'success': success,
'content': promise_content, 'content': promise_content,
'failure_amount': failure_count, 'failure_amount': failure_count,
'periodicity': periodicity} 'periodicity': periodicity,
'paths': PROMISE_PATHS}
with open(os.path.join(promise_dir, promise_name), 'w') as f: with open(os.path.join(promise_dir, promise_name), 'w') as f:
f.write(_promise_content) f.write(_promise_content)
...@@ -599,7 +617,7 @@ class InstanceForTest(object): ...@@ -599,7 +617,7 @@ class InstanceForTest(object):
class SoftwareForTest(object): class SoftwareForTest(object):
""" """
Class to prepare and simulate software. Class to prepare and simulate software.
each instance has a sotfware attributed each instance has a software attributed
""" """
def __init__(self, software_root, name=''): def __init__(self, software_root, name=''):
""" """
...@@ -2297,6 +2315,7 @@ exit 1 ...@@ -2297,6 +2315,7 @@ exit 1
self.assertFalse(os.path.exists(promise_file)) self.assertFalse(os.path.exists(promise_file))
self.assertTrue(instance.error) self.assertTrue(instance.error)
class TestSlapgridDestructionLock(MasterMixin, unittest.TestCase): class TestSlapgridDestructionLock(MasterMixin, unittest.TestCase):
def test_retention_lock(self): def test_retention_lock(self):
""" """
...@@ -3999,9 +4018,6 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase): ...@@ -3999,9 +4018,6 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase):
self.assertEqual('success', result["result"]["message"]) self.assertEqual('success', result["result"]["message"])
def test_one_succeeding_one_timing_out_promises(self): def test_one_succeeding_one_timing_out_promises(self):
computer = ComputerForTest(self.software_root, self.instance_root) computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler): with httmock.HTTMock(computer.request_handler):
...@@ -4095,6 +4111,58 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase): ...@@ -4095,6 +4111,58 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase):
".slapgrid/promise/result/fail.status.json"))) ".slapgrid/promise/result/fail.status.json")))
class TestSlapgridPluginPromiseWithInstancePython(TestSlapgridPromiseWithMaster):
expect_plugin = False
def setPython(self):
self.python_called = os.path.join(self.software_root, 'called')
wrapper = """#!/bin/sh
touch %s
exec %s "$@"
""" % (self.python_called, sys.executable)
path = os.path.join(self.software_root, 'python')
with open(path, 'w') as f:
f.write(wrapper)
os.chmod(path, 0o755)
return path
def patchBuildoutSetter(self):
cls = SoftwareForTest
attr = 'setBuildout'
orig = getattr(cls, attr)
def setBuildout(soft):
buildout = "#!" + self.setPython()
orig(soft, buildout)
self.addCleanup(setattr, cls, attr, orig)
setattr(cls, attr, setBuildout)
def patchPluginSetter(self):
cls = InstanceForTest
attr = 'setPluginPromise'
orig = getattr(cls, attr)
def setPluginPromise(inst, *args, **kwargs):
self.expect_plugin = inst.requested_state == 'started'
return orig(inst, *args, **kwargs)
self.addCleanup(setattr, cls, attr, orig)
setattr(cls, attr, setPluginPromise)
def setMock(self):
self.patchBuildoutSetter()
self.patchPluginSetter()
def tearDown(self):
try:
os.remove(self.python_called)
called = True
except OSError as e:
if e.errno != errno.ENOENT:
raise
called = False
finally:
super(TestSlapgridPluginPromiseWithInstancePython, self).tearDown()
self.assertEqual(self.expect_plugin, called)
class TestSVCBackend(unittest.TestCase): class TestSVCBackend(unittest.TestCase):
"""Tests for supervisor backend. """Tests for supervisor backend.
""" """
......
...@@ -95,6 +95,18 @@ def mkdir_p(path, mode=0o700): ...@@ -95,6 +95,18 @@ def mkdir_p(path, mode=0o700):
raise raise
def listifdir(path):
"""
Like listdir, but returns an empty tuple if the path is not a directory.
"""
try:
return os.listdir(path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
return ()
def chownDirectory(path, uid, gid): def chownDirectory(path, uid, gid):
if os.getuid() != 0: if os.getuid() != 0:
# we are probably inside of a webrunner # we are probably inside of a webrunner
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment