From 38cd519a98aa5d9b48447f9e0919ff439ad2960e Mon Sep 17 00:00:00 2001
From: Alain Takoudjou <talino@tiolive.com>
Date: Fri, 12 Oct 2012 10:42:58 +0200
Subject: [PATCH] Update BOINC, allows to install application with
slapparameters
---
slapos/recipe/boinc/__init__.py | 66 +++++++++++++++++++++-----
slapos/recipe/boinc/configure.py | 80 ++++++++++++++++++++------------
software/boinc/software.cfg | 2 +-
stack/boinc/buildout.cfg | 4 +-
stack/boinc/instance-boinc.cfg | 3 +-
5 files changed, 109 insertions(+), 46 deletions(-)
diff --git a/slapos/recipe/boinc/__init__.py b/slapos/recipe/boinc/__init__.py
index 5d4ff9ff3..2c71a8ba7 100644
--- a/slapos/recipe/boinc/__init__.py
+++ b/slapos/recipe/boinc/__init__.py
@@ -29,6 +29,7 @@ import os
import subprocess
import pwd
import signal
+import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational boinc architecture."""
@@ -208,11 +209,51 @@ class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter"""
+ def downloadFiles(self):
+ """This is used to download app files if necessary and update options values"""
+ for key in ('template-result', 'template-wu', 'input-file', 'binary'):
+ option = self.options[key].strip()
+ if option and (option.startswith('http') or option.startswith('ftp')):
+ #download the specified file
+ cache = os.path.join(self.options['home'].strip(), 'temp')
+ downloader = zc.buildout.download.Download(self.buildout['buildout'],
+ hash_name=True, cache=cache)
+ path, _ = downloader(option, md5sum=None)
+ mode = 0600
+ if key == 'binary':
+ mode = 0700
+ os.chmod(path, mode)
+ self.options[key] = path
+
+ def checkOptions(self):
+ """Check if parameter send is valid to install or update application"""
+ if not self.options['app-name'].strip() or \
+ not self.options['version'].strip():
+ return False
+ self.appname = self.options['app-name'].strip()
+ self.version = self.options['version'].strip()
+ #for non exist application, check if parameter is complete
+ appdir = os.path.join(self.options['installroot'].strip(), 'apps',
+ self.options['app-name'].strip(),
+ self.options['version'].strip())
+ if not os.path.exists(appdir):
+ if not self.options['template-result'].strip() or not self.options['binary'].strip() \
+ or not self.options['input-file'].strip() or not self.options['template-wu'].strip() \
+ or not self.options['wu-number'].strip() or not self.options['platform'].strip():
+ print "Invalid argement values...operation cancelled"
+ return False
+ #write application to install
+ toInstall = open(os.path.join(self.options['home'].strip(),
+ '.install_' + self.appname + self.version), 'w')
+ toInstall.write('install or update')
+ toInstall.close()
+ return True
def install(self):
- if self.options['app-name'].strip() == '' or \
- self.options['version'].strip() == '':
- #don't deploy empty application...skipped
+ self.appname = ''
+ self.version = ''
+ if not self.checkOptions():
+ #don't deploy empty or invalid application...skipped
return []
path_list = []
package = self.options['boinc'].strip()
@@ -221,6 +262,7 @@ class App(GenericBaseRecipe):
developegg = self.options['develop-egg'].strip()
python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
home = self.options['home'].strip()
+ user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip()
svn = self.options['svn-binary'].strip()
for f in os.listdir(developegg):
@@ -239,26 +281,26 @@ class App(GenericBaseRecipe):
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
- user=self.options['user']))
+ user=user))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
+ #If useful, download necessary files and update options path
+ self.downloadFiles()
start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip()
- version = self.options['version'].strip()
platform = self.options['platform'].strip()
apps_dir = os.path.join(installroot, 'apps')
- appname = self.options['app-name'].strip()
- bin_name = appname +"_"+ version +"_"+ \
+ bin_name = self.appname +"_"+ self.version +"_"+ \
platform + self.options['extension'].strip()
- application = os.path.join(apps_dir, appname, version, platform)
+ application = os.path.join(apps_dir, self.appname, self.version, platform)
wrapperdir = self.options['wrapper-dir'].strip()
project = self.options['project'].strip()
parameter = dict(installroot=installroot, project=project,
- appname=appname, binary_name=bin_name,
- version=version, platform=platform,
+ appname=self.appname, binary_name=bin_name,
+ version=self.version, platform=platform,
application=application, environment=environment,
start_boinc=start_boinc,
wu_number=int(self.options['wu-number'].strip()),
@@ -266,10 +308,10 @@ class App(GenericBaseRecipe):
t_wu=self.options['template-wu'].strip(),
t_input=self.options['input-file'].strip(),
binary=self.options['binary'].strip(),
- bash=bash,
+ bash=bash, home_dir=home,
)
deploy_app = self.createPythonScript(
- os.path.join(wrapperdir, appname),
+ os.path.join(wrapperdir, self.appname),
'%s.configure.deployApp' % __name__, parameter
)
path_list.append(deploy_app)
diff --git a/slapos/recipe/boinc/configure.py b/slapos/recipe/boinc/configure.py
index a05191fdb..dbc4f82d3 100644
--- a/slapos/recipe/boinc/configure.py
+++ b/slapos/recipe/boinc/configure.py
@@ -53,6 +53,7 @@ def checkMysql(args):
print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2)
+
def checkFile(file, stime):
"""Loop until 'file' is created (exist)"""
while True:
@@ -63,6 +64,7 @@ def checkFile(file, stime):
else:
break
+
def services(args):
"""This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..."
@@ -115,6 +117,7 @@ def services(args):
writeFile(args['service_status'], "started")
+
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
if args['drop_install']:
@@ -130,54 +133,68 @@ def restart_boinc(args):
writeFile(args['start_boinc'], "started")
print "Done."
+
def deployApp(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
print "Cheking if needed to install %s..." % args['appname']
+ install_request_file = os.path.join(args['home_dir'],
+ '.install_' + args['appname'] + args['version'])
+ if not os.path.exists(install_request_file):
+ print "No install or update request for %s version %s..." % (
+ args['appname'], args['version'])
+ return
+ os.unlink(install_request_file)
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
- dropapp = False
+ newInstall = False
if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip())
- if args['previous_wu'] >= args['wu_number']:
- print args['appname'] + " version " + args['version'] + " is already installed in this Boinc instance... skipped"
- return
- else:
+ if args['previous_wu'] < args['wu_number']:
print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number'])
else:
args['previous_wu'] = 0
- dropapp = True
+ newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
print "setup directories..."
args['inputfile'] = os.path.join(args['installroot'], 'download',
- args['appname'] + '_input')
+ args['appname'] + args['version'] + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
- t_result = os.path.join(args['templates'], args['appname']+'_result')
- t_wu = os.path.join(args['templates'], args['appname']+'_wu')
+ t_result = os.path.join(args['templates'],
+ args['appname'] + args['version'] + '_result')
+ t_wu = os.path.join(args['templates'],
+ args['appname'] + args['version'] + '_wu')
+ binary = os.path.join(args['application'], args['binary_name'])
if not os.path.exists(base_app):
os.mkdir(base_app)
- if dropapp:
+ if newInstall:
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
- if not os.path.exists(args['templates']):
- os.mkdir(args['templates'])
- else:
- if os.path.exists(t_result):
- os.unlink(t_result)
- if os.path.exists(t_result):
- os.unlink(t_wu)
+ if not os.path.exists(args['templates']):
+ os.mkdir(args['templates'])
+ if args['t_result']:
+ if os.path.exists(t_result):
+ os.unlink(t_result)
shutil.copy(args['t_result'], t_result)
+ if args['t_wu']:
+ if os.path.exists(t_wu):
+ os.unlink(t_wu)
shutil.copy(args['t_wu'], t_wu)
- if not os.path.exists(args['inputfile']):
- os.symlink(args['t_input'], args['inputfile'])
- shutil.copy(args['binary'], os.path.join(args['application'],
- args['binary_name']))
+ if args['t_input']:
+ if os.path.exists(args['inputfile']):
+ os.unlink(args['inputfile'])
+ os.symlink(args['t_input'], args['inputfile'])
+ if args['binary'] and args['platform']:
+ if os.path.exists(binary):
+ os.unlink(binary)
+ shutil.copy(args['binary'], binary)
+ if newInstall:
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml')
@@ -185,19 +202,18 @@ def deployApp(args):
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
+ if args['binary'] and args['platform']:
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
- output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w')
- p_sign = subprocess.Popen([sign, os.path.join(args['application'],
- args['binary_name']), privateKeyFile], stdout=output,
+ output = open(binary + '.sig', 'w')
+ p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
- #END if drop-app HERE
print "Running xadd script..."
env = os.environ
@@ -231,19 +247,21 @@ def deployApp(args):
print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number']))
+
def create_wu(args, env):
- t_result = "templates/" + args['appname'] + '_result'
- t_wu = "templates/" + args['appname'] + '_wu'
+ t_result = "templates/" + args['appname'] + args['version'] + '_result'
+ t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1',
- args['appname']+'_input']
+ args['appname'] + args['version'] + '_input']
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
startProcess(launch_args, env, args['installroot'])
+
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
@@ -254,13 +272,14 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
return False
return True
+
def runCmd(args):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml')
checkFile(client_config, 5)
time.sleep(10)
#Scan client state xml to find client ipv4 adress
- host = result = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
+ host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
print "Run boinccmd with host at %s " % args['base_cmd'][2]
@@ -271,7 +290,8 @@ def runCmd(args):
#Load or reload cc_config file
startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
+
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
- f.close()
\ No newline at end of file
+ f.close()
diff --git a/software/boinc/software.cfg b/software/boinc/software.cfg
index 033339902..d411ad00f 100644
--- a/software/boinc/software.cfg
+++ b/software/boinc/software.cfg
@@ -38,7 +38,7 @@ download-only = true
filename = upper_case
#Application configuration
app-name = upper_case
-version = 1.0
+version = 1.00
exec-extension =
#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
platform = x86_64-pc-linux-gnu
diff --git a/stack/boinc/buildout.cfg b/stack/boinc/buildout.cfg
index 6c18b691e..898827a8f 100644
--- a/stack/boinc/buildout.cfg
+++ b/stack/boinc/buildout.cfg
@@ -41,7 +41,7 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg
-md5sum = b85d32e9509e960459c2ba1e47741838
+md5sum = 003bff525faa1e63913fa5f38c18becd
mode = 0644
#Template for deploying MySQL Database Server
@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
mode = 0644
filename = apache.in
-md5sum = 2d3efa25bf1cf6a673fb2cef13ae0529
+md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Local development
diff --git a/stack/boinc/instance-boinc.cfg b/stack/boinc/instance-boinc.cfg
index 7f92a8c80..a7dc582dd 100644
--- a/stack/boinc/instance-boinc.cfg
+++ b/stack/boinc/instance-boinc.cfg
@@ -218,7 +218,8 @@ mysql-port = $${stunnel:local-port}
[slap-application]
<= boinc-server
recipe = slapos.cookbook:boinc.app
-#appname and version is require to update wu-number
+#appname and version is require to update any existing application
+#otherwise, the recipe would try to install a new one
app-name = $${slap-parameter:app-name}
version = $${slap-parameter:version}
wu-number = $${slap-parameter:wu-number}
--
2.30.9