Commit 38cd519a authored by Alain Takoudjou's avatar Alain Takoudjou

Update BOINC, allows to install application with slapparameters

parent b56390cd
...@@ -29,6 +29,7 @@ import os ...@@ -29,6 +29,7 @@ import os
import subprocess import subprocess
import pwd import pwd
import signal import signal
import zc.buildout
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
"""Deploy a fully operational boinc architecture.""" """Deploy a fully operational boinc architecture."""
...@@ -208,11 +209,51 @@ class App(GenericBaseRecipe): ...@@ -208,11 +209,51 @@ class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc """This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter""" Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self):
"""This is used to download app files if necessary and update options values"""
for key in ('template-result', 'template-wu', 'input-file', 'binary'):
option = self.options[key].strip()
if option and (option.startswith('http') or option.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'temp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(option, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
os.chmod(path, mode)
self.options[key] = path
def checkOptions(self):
"""Check if parameter send is valid to install or update application"""
if not self.options['app-name'].strip() or \
not self.options['version'].strip():
return False
self.appname = self.options['app-name'].strip()
self.version = self.options['version'].strip()
#for non exist application, check if parameter is complete
appdir = os.path.join(self.options['installroot'].strip(), 'apps',
self.options['app-name'].strip(),
self.options['version'].strip())
if not os.path.exists(appdir):
if not self.options['template-result'].strip() or not self.options['binary'].strip() \
or not self.options['input-file'].strip() or not self.options['template-wu'].strip() \
or not self.options['wu-number'].strip() or not self.options['platform'].strip():
print "Invalid argement values...operation cancelled"
return False
#write application to install
toInstall = open(os.path.join(self.options['home'].strip(),
'.install_' + self.appname + self.version), 'w')
toInstall.write('install or update')
toInstall.close()
return True
def install(self): def install(self):
if self.options['app-name'].strip() == '' or \ self.appname = ''
self.options['version'].strip() == '': self.version = ''
#don't deploy empty application...skipped if not self.checkOptions():
#don't deploy empty or invalid application...skipped
return [] return []
path_list = [] path_list = []
package = self.options['boinc'].strip() package = self.options['boinc'].strip()
...@@ -221,6 +262,7 @@ class App(GenericBaseRecipe): ...@@ -221,6 +262,7 @@ class App(GenericBaseRecipe):
developegg = self.options['develop-egg'].strip() developegg = self.options['develop-egg'].strip()
python_path = boinc_egg + ":" + os.environ['PYTHONPATH'] python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
home = self.options['home'].strip() home = self.options['home'].strip()
user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip() perl = self.options['perl-binary'].strip()
svn = self.options['svn-binary'].strip() svn = self.options['svn-binary'].strip()
for f in os.listdir(developegg): for f in os.listdir(developegg):
...@@ -239,26 +281,26 @@ class App(GenericBaseRecipe): ...@@ -239,26 +281,26 @@ class App(GenericBaseRecipe):
self.substituteTemplate(self.getTemplateFilename('sed_update.in'), self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip(), dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(), uldl_pid=self.options['apache-pid'].strip(),
user=self.options['user'])) user=user))
) )
path_list.append(sh_script) path_list.append(sh_script)
os.chmod(bash , 0700) os.chmod(bash , 0700)
#If useful, download necessary files and update options path
self.downloadFiles()
start_boinc = os.path.join(home, '.start_boinc') start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip() installroot = self.options['installroot'].strip()
version = self.options['version'].strip()
platform = self.options['platform'].strip() platform = self.options['platform'].strip()
apps_dir = os.path.join(installroot, 'apps') apps_dir = os.path.join(installroot, 'apps')
appname = self.options['app-name'].strip() bin_name = self.appname +"_"+ self.version +"_"+ \
bin_name = appname +"_"+ version +"_"+ \
platform + self.options['extension'].strip() platform + self.options['extension'].strip()
application = os.path.join(apps_dir, appname, version, platform) application = os.path.join(apps_dir, self.appname, self.version, platform)
wrapperdir = self.options['wrapper-dir'].strip() wrapperdir = self.options['wrapper-dir'].strip()
project = self.options['project'].strip() project = self.options['project'].strip()
parameter = dict(installroot=installroot, project=project, parameter = dict(installroot=installroot, project=project,
appname=appname, binary_name=bin_name, appname=self.appname, binary_name=bin_name,
version=version, platform=platform, version=self.version, platform=platform,
application=application, environment=environment, application=application, environment=environment,
start_boinc=start_boinc, start_boinc=start_boinc,
wu_number=int(self.options['wu-number'].strip()), wu_number=int(self.options['wu-number'].strip()),
...@@ -266,10 +308,10 @@ class App(GenericBaseRecipe): ...@@ -266,10 +308,10 @@ class App(GenericBaseRecipe):
t_wu=self.options['template-wu'].strip(), t_wu=self.options['template-wu'].strip(),
t_input=self.options['input-file'].strip(), t_input=self.options['input-file'].strip(),
binary=self.options['binary'].strip(), binary=self.options['binary'].strip(),
bash=bash, bash=bash, home_dir=home,
) )
deploy_app = self.createPythonScript( deploy_app = self.createPythonScript(
os.path.join(wrapperdir, appname), os.path.join(wrapperdir, self.appname),
'%s.configure.deployApp' % __name__, parameter '%s.configure.deployApp' % __name__, parameter
) )
path_list.append(deploy_app) path_list.append(deploy_app)
......
...@@ -53,6 +53,7 @@ def checkMysql(args): ...@@ -53,6 +53,7 @@ def checkMysql(args):
print "Could not connect to MySQL database... sleep for 2 secondes" print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2) time.sleep(2)
def checkFile(file, stime): def checkFile(file, stime):
"""Loop until 'file' is created (exist)""" """Loop until 'file' is created (exist)"""
while True: while True:
...@@ -63,6 +64,7 @@ def checkFile(file, stime): ...@@ -63,6 +64,7 @@ def checkFile(file, stime):
else: else:
break break
def services(args): def services(args):
"""This function configure a new installed boinc project instance""" """This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..." print "Checking if needed to install or reinstall Boinc-server..."
...@@ -115,6 +117,7 @@ def services(args): ...@@ -115,6 +117,7 @@ def services(args):
writeFile(args['service_status'], "started") writeFile(args['service_status'], "started")
def restart_boinc(args): def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service""" """Stop (if currently is running state) and start all Boinc service"""
if args['drop_install']: if args['drop_install']:
...@@ -130,54 +133,68 @@ def restart_boinc(args): ...@@ -130,54 +133,68 @@ def restart_boinc(args):
writeFile(args['start_boinc'], "started") writeFile(args['start_boinc'], "started")
print "Done." print "Done."
def deployApp(args): def deployApp(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance""" """Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
print "Cheking if needed to install %s..." % args['appname'] print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
return
os.unlink(install_request_file)
token = os.path.join(args['installroot'], "." + args['appname'] + args['version']) token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
dropapp = False newInstall = False
if os.path.exists(token): if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip()) args['previous_wu'] = int(open(token, 'r').read().strip())
if args['previous_wu'] >= args['wu_number']: if args['previous_wu'] < args['wu_number']:
print args['appname'] + " version " + args['version'] + " is already installed in this Boinc instance... skipped"
return
else:
print args['appname'] + " Work units will be updated from %s to %s" % ( print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number']) args['previous_wu'], args['wu_number'])
else: else:
args['previous_wu'] = 0 args['previous_wu'] = 0
dropapp = True newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started) #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3) checkFile(args['start_boinc'], 3)
print "setup directories..." print "setup directories..."
args['inputfile'] = os.path.join(args['installroot'], 'download', args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname'] + '_input') args['appname'] + args['version'] + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname']) base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version']) base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates') args['templates'] = os.path.join(args['installroot'], 'templates')
t_result = os.path.join(args['templates'], args['appname']+'_result') t_result = os.path.join(args['templates'],
t_wu = os.path.join(args['templates'], args['appname']+'_wu') args['appname'] + args['version'] + '_result')
t_wu = os.path.join(args['templates'],
args['appname'] + args['version'] + '_wu')
binary = os.path.join(args['application'], args['binary_name'])
if not os.path.exists(base_app): if not os.path.exists(base_app):
os.mkdir(base_app) os.mkdir(base_app)
if dropapp: if newInstall:
if os.path.exists(base_app_version): if os.path.exists(base_app_version):
shutil.rmtree(base_app_version) shutil.rmtree(base_app_version)
os.mkdir(base_app_version) os.mkdir(base_app_version)
os.mkdir(args['application']) os.mkdir(args['application'])
if not os.path.exists(args['templates']): if not os.path.exists(args['templates']):
os.mkdir(args['templates']) os.mkdir(args['templates'])
else: if args['t_result']:
if os.path.exists(t_result): if os.path.exists(t_result):
os.unlink(t_result) os.unlink(t_result)
if os.path.exists(t_result):
os.unlink(t_wu)
shutil.copy(args['t_result'], t_result) shutil.copy(args['t_result'], t_result)
if args['t_wu']:
if os.path.exists(t_wu):
os.unlink(t_wu)
shutil.copy(args['t_wu'], t_wu) shutil.copy(args['t_wu'], t_wu)
if not os.path.exists(args['inputfile']): if args['t_input']:
if os.path.exists(args['inputfile']):
os.unlink(args['inputfile'])
os.symlink(args['t_input'], args['inputfile']) os.symlink(args['t_input'], args['inputfile'])
shutil.copy(args['binary'], os.path.join(args['application'], if args['binary'] and args['platform']:
args['binary_name'])) if os.path.exists(binary):
os.unlink(binary)
shutil.copy(args['binary'], binary)
if newInstall:
print "Adding '" + args['appname'] + "' to project.xml..." print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..." print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml') project_xml = os.path.join(args['installroot'], 'project.xml')
...@@ -185,19 +202,18 @@ def deployApp(args): ...@@ -185,19 +202,18 @@ def deployApp(args):
sed_args = [args['bash'], args['appname'], args['installroot']] sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args) startProcess(sed_args)
if args['binary'] and args['platform']:
print "Sign the application binary..." print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable') sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private') privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w') output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, os.path.join(args['application'], p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
args['binary_name']), privateKeyFile], stdout=output,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
result = p_sign.communicate()[0] result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0: if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return return
output.close() output.close()
#END if drop-app HERE
print "Running xadd script..." print "Running xadd script..."
env = os.environ env = os.environ
...@@ -231,19 +247,21 @@ def deployApp(args): ...@@ -231,19 +247,21 @@ def deployApp(args):
print "Boinc Application deployment is done... writing end signal file..." print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number'])) writeFile(token, str(args['wu_number']))
def create_wu(args, env): def create_wu(args, env):
t_result = "templates/" + args['appname'] + '_result' t_result = "templates/" + args['appname'] + args['version'] + '_result'
t_wu = "templates/" + args['appname'] + '_wu' t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'), launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '', '--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result, '--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1', '--min_quorum', '1', '--target_nresults', '1',
args['appname']+'_input'] args['appname'] + args['version'] + '_input']
for i in range(args['previous_wu'], args['wu_number']): for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1) print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete' launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
startProcess(launch_args, env, args['installroot']) startProcess(launch_args, env, args['installroot'])
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE): def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout, process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env, stderr=subprocess.STDOUT, env=env,
...@@ -254,13 +272,14 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE): ...@@ -254,13 +272,14 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
return False return False
return True return True
def runCmd(args): def runCmd(args):
"""Wait for Boinc Client started and run boinc cmd""" """Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml') client_config = os.path.join(args['installdir'], 'client_state.xml')
checkFile(client_config, 5) checkFile(client_config, 5)
time.sleep(10) time.sleep(10)
#Scan client state xml to find client ipv4 adress #Scan client state xml to find client ipv4 adress
host = result = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>", host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1) open(client_config, 'r').read()).group(1)
args['base_cmd'][2] = host + ':' + args['base_cmd'][2] args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
print "Run boinccmd with host at %s " % args['base_cmd'][2] print "Run boinccmd with host at %s " % args['base_cmd'][2]
...@@ -271,6 +290,7 @@ def runCmd(args): ...@@ -271,6 +290,7 @@ def runCmd(args):
#Load or reload cc_config file #Load or reload cc_config file
startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir']) startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
def writeFile(file, content): def writeFile(file, content):
f = open(file, 'w') f = open(file, 'w')
f.write(content) f.write(content)
......
...@@ -38,7 +38,7 @@ download-only = true ...@@ -38,7 +38,7 @@ download-only = true
filename = upper_case filename = upper_case
#Application configuration #Application configuration
app-name = upper_case app-name = upper_case
version = 1.0 version = 1.00
exec-extension = exec-extension =
#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms #Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
platform = x86_64-pc-linux-gnu platform = x86_64-pc-linux-gnu
......
...@@ -41,7 +41,7 @@ eggs = ...@@ -41,7 +41,7 @@ eggs =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg output = ${buildout:directory}/template-boinc.cfg
md5sum = b85d32e9509e960459c2ba1e47741838 md5sum = 003bff525faa1e63913fa5f38c18becd
mode = 0644 mode = 0644
#Template for deploying MySQL Database Server #Template for deploying MySQL Database Server
...@@ -57,7 +57,7 @@ recipe = slapos.recipe.download ...@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename} url = ${:_profile_base_location_}/template/${:filename}
mode = 0644 mode = 0644
filename = apache.in filename = apache.in
md5sum = 2d3efa25bf1cf6a673fb2cef13ae0529 md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Local development # Local development
......
...@@ -218,7 +218,8 @@ mysql-port = $${stunnel:local-port} ...@@ -218,7 +218,8 @@ mysql-port = $${stunnel:local-port}
[slap-application] [slap-application]
<= boinc-server <= boinc-server
recipe = slapos.cookbook:boinc.app recipe = slapos.cookbook:boinc.app
#appname and version is require to update wu-number #appname and version is require to update any existing application
#otherwise, the recipe would try to install a new one
app-name = $${slap-parameter:app-name} app-name = $${slap-parameter:app-name}
version = $${slap-parameter:version} version = $${slap-parameter:version}
wu-number = $${slap-parameter:wu-number} wu-number = $${slap-parameter:wu-number}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment