Skip to content

S
slapos-mynij-dev
Commit 4bb47083 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

factorize the code of boinc recipe

parent 28bed57e
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 103 deletions
slapos/recipe/boinc/__init__.py
View file @ 4bb47083
......@@ -103,7 +103,7 @@ class Recipe(GenericBaseRecipe):
#Define environment variable here
python = os.path.join(self.home, 'bin/python')
python_path = self.boinc_egg + ":" + os.environ['PYTHONPATH']
python_path = self.boinc_egg
if not os.path.exists(python):
os.symlink(self.pythonbin, python)
for f in os.listdir(self.developegg):
......@@ -112,8 +112,8 @@ class Recipe(GenericBaseRecipe):
python_path += ":" + dir
bin_dir = os.path.join(self.home, 'bin')
environment = dict(
PATH=self.svn + ':' + bin_dir + ':' + self.perl + ':' + os.environ['PATH'],
PYTHONPATH=python_path,
PATH=os.pathsep.join([self.svn, bin_dir, self.perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#Generate wrapper for php
......@@ -134,7 +134,7 @@ class Recipe(GenericBaseRecipe):
dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
mysql_user=self.username, mysql_password=self.password,
database=self.database,
file_status=file_status, python_path=python_path
file_status=file_status, environment=environment
)
)
......@@ -160,6 +160,18 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(install_wrapper)
#generate sh script for project configuration
bash = os.path.join(self.home, 'bin', 'project_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('project_config.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
user=slapuser, fullname=self.fullname,
copyright=self.copyright, installroot=self.installroot))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#After make_project run configure_script to perform and restart apache php services
service_status = os.path.join(self.home, '.start_service')
parameter = dict(
......@@ -171,10 +183,8 @@ class Recipe(GenericBaseRecipe):
xadd=os.path.join(self.installroot, 'bin/xadd'),
environment=environment,
service_status=service_status,
project=niceprojectname,
fullname=self.fullname,
copyright=self.copyright,
drop_install=drop_install
drop_install=drop_install,
sedconfig=bash
)
start_service = self.createPythonScript(
os.path.join(self.wrapperdir, 'config_project'),
......@@ -191,8 +201,8 @@ class Recipe(GenericBaseRecipe):
installroot=self.installroot, drop_install=drop_install,
mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
mysql_user=self.username, mysql_password=self.password,
database=self.database, PATH=environment['PATH'],
python_path=python_path, start_boinc=start_boinc)
database=self.database, environment=environment,
start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'),
'%s.configure.restart_boinc' % __name__,
......@@ -211,14 +221,14 @@ class App(GenericBaseRecipe):
def downloadFiles(self):
"""This is used to download app files if necessary and update options values"""
for key in ('template-result', 'template-wu', 'input-file', 'binary'):
option = self.options[key].strip()
if option and (option.startswith('http') or option.startswith('ftp')):
for key in ('input-file', 'template-result', 'template-wu', 'binary'):
param = self.options[key].strip()
if param and (param.startswith('http') or param.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'temp')
cache = os.path.join(self.options['home'].strip(), 'tmp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(option, md5sum=None)
path, _ = downloader(param, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
......@@ -243,8 +253,9 @@ class App(GenericBaseRecipe):
print "Invalid argement values...operation cancelled"
return False
#write application to install
toInstall = open(os.path.join(self.options['home'].strip(),
'.install_' + self.appname + self.version), 'w')
request_file = os.path.join(self.options['home'].strip(),
'.install_' + self.appname + self.version)
toInstall = open(request_file, 'w')
toInstall.write('install or update')
toInstall.close()
return True
......@@ -258,9 +269,8 @@ class App(GenericBaseRecipe):
path_list = []
package = self.options['boinc'].strip()
#Define environment variable here
boinc_egg = os.path.join(package, 'lib/python2.7/site-packages')
developegg = self.options['develop-egg'].strip()
python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
python_path = os.path.join(package, 'lib/python2.7/site-packages')
home = self.options['home'].strip()
user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip()
......@@ -271,17 +281,15 @@ class App(GenericBaseRecipe):
python_path += ":" + dir
bin_dir = os.path.join(home, 'bin')
environment = dict(
PATH=svn + ':' + bin_dir + ':' + perl + ':' + os.environ['PATH'],
PYTHONPATH=python_path,
PATH=os.pathsep.join([svn, bin_dir, perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#generate project.xml and config.xml script updater
bash = os.path.join(home, 'bin', 'update_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
user=user))
dict(dash=self.options['dash'].strip()))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
......@@ -311,7 +319,7 @@ class App(GenericBaseRecipe):
bash=bash, home_dir=home,
)
deploy_app = self.createPythonScript(
os.path.join(wrapperdir, self.appname),
os.path.join(wrapperdir, 'boinc_app'),
'%s.configure.deployApp' % __name__, parameter
)
path_list.append(deploy_app)
......
slapos/recipe/boinc/configure.py
View file @ 4bb47083
......@@ -31,9 +31,10 @@ import subprocess
import time
import shutil
import re
import filecmp
def checkMysql(args):
sys.path += args['python_path'].split(':')
sys.path += args['environment']['PYTHONPATH'].split(':')
import MySQLdb
#Sleep until mysql server becomes available
while True:
......@@ -65,6 +66,61 @@ def checkFile(file, stime):
break
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
if args['drop_install']:
checkFile(args['service_status'], 3)
else:
checkMysql(args)
print "Restart Boinc..."
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
os.system(binstop)
os.system(binstart)
writeFile(args['start_boinc'], "started")
print "Done."
def check_installRequest(args):
print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
return False
os.unlink(install_request_file)
return True
def copy_file(source, dest):
""""Copy file with source to dest with auto replace
return True if file has been copied and dest ha been replaced
"""
result = False
if source and os.path.exists(source):
if os.path.exists(dest):
if filecmp.cmp(dest, source):
return False
os.unlink(dest)
result = True
shutil.copy(source, dest)
return result
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute executable.\nThe error was: %s" % result
return False
return True
def services(args):
"""This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..."
......@@ -81,11 +137,11 @@ def services(args):
if not startProcess(htpwd_args):
return
print "Running xadd script..."
print "execute script xadd..."
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
if not startProcess([args['xadd']], env):
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Update files and directories permissions..."
upload = os.path.join(args['installroot'], 'upload')
......@@ -103,11 +159,8 @@ def services(args):
os.system("chmod -R o+r " + inc)
os.system("chmod o+x " + languages)
os.system("chmod o+x " + compiled)
os.system("sed -i '/remove the die/d' %s" % forum_file)
subprocess.Popen(["sed -i 's#REPLACE WITH PROJECT NAME#%s#' %s" % (args['fullname'],
project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0]
subprocess.Popen(["sed -i 's#REPLACE WITH COPYRIGHT HOLDER#%s#' %s" % (args['copyright'],
project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0]
sed_args = [args['sedconfig']]
startProcess(sed_args)
#Execute php create_forum.php...
print "Boinc Forum: Execute php create_forum.php..."
......@@ -118,32 +171,10 @@ def services(args):
writeFile(args['service_status'], "started")
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
if args['drop_install']:
checkFile(args['service_status'], 3)
else:
checkMysql(args)
print "Restart Boinc..."
os.environ['PATH'] = args['PATH']
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
os.system(binstop)
os.system(binstart)
writeFile(args['start_boinc'], "started")
print "Done."
def deployApp(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
if not check_installRequest(args):
return
os.unlink(install_request_file)
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
newInstall = False
if os.path.exists(token):
......@@ -156,18 +187,23 @@ def deployApp(args):
newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
print "setup directories..."
numversion = args['version'].replace('.', '')
args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname'] + args['version'] + '_input')
args['appname'] + numversion + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
t_result = os.path.join(args['templates'],
args['appname'] + args['version'] + '_result')
args['appname'] + numversion + '_result')
t_wu = os.path.join(args['templates'],
args['appname'] + args['version'] + '_wu')
args['appname'] + numversion + '_wu')
binary = os.path.join(args['application'], args['binary_name'])
signBin = False
if not os.path.exists(base_app):
os.mkdir(base_app)
if newInstall:
......@@ -177,32 +213,24 @@ def deployApp(args):
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
if args['t_result']:
if os.path.exists(t_result):
os.unlink(t_result)
shutil.copy(args['t_result'], t_result)
if args['t_wu']:
if os.path.exists(t_wu):
os.unlink(t_wu)
shutil.copy(args['t_wu'], t_wu)
copy_file(args['t_result'], t_result)
copy_file(args['t_wu'], t_wu)
signBin = copy_file(args['binary'], binary)
if args['t_input']:
if os.path.exists(args['inputfile']):
os.unlink(args['inputfile'])
os.symlink(args['t_input'], args['inputfile'])
if args['binary'] and args['platform']:
if os.path.exists(binary):
os.unlink(binary)
shutil.copy(args['binary'], binary)
if newInstall:
project_xml = os.path.join(args['installroot'], 'project.xml')
findapp = re.search("<name>(%s)</name>" % args['appname'],
open(project_xml, 'r').read())
if not findapp:
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml')
config_xml = os.path.join(args['installroot'], 'config.xml')
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
if args['binary'] and args['platform']:
if signBin:
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
......@@ -215,13 +243,10 @@ def deployApp(args):
return
output.close()
print "Running xadd script..."
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
print "execute script xadd..."
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Running script bin/update_versions..."
updt_version = os.path.join(args['installroot'], 'bin/update_versions')
p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
......@@ -240,7 +265,6 @@ def deployApp(args):
print "Restart Boinc..."
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
sys.environ = env
os.system(binstop)
os.system(binstart)
......@@ -249,30 +273,21 @@ def deployApp(args):
def create_wu(args, env):
t_result = "templates/" + args['appname'] + args['version'] + '_result'
t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
"""Create or update number of work unit for an existing boinc application"""
numversion = args['version'].replace('.', '')
t_result = "templates/" + args['appname'] + numversion + '_result'
t_wu = "templates/" + args['appname'] + numversion + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1',
args['appname'] + args['version'] + '_input']
args['appname'] + numversion + '_input']
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
launch_args[4] = args['appname'] + str(i+1) + numversion + '_nodelete'
startProcess(launch_args, env, args['installroot'])
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute executable.\nThe error was: %s" % result
return False
return True
def runCmd(args):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml')
......
slapos/recipe/boinc/template/project_config.in 0 → 100644
View file @ 4bb47083
#!%(dash)s
sed -i "/remove the die/d" %(installroot)s/html/ops/create_forums.php
sed -i "s#REPLACE WITH PROJECT NAME#%(fullname)s#" %(installroot)s/html/project/project.inc
sed -i "s#REPLACE WITH COPYRIGHT HOLDER#%(copyright)s#" %(installroot)s/html/project/project.inc
sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" %(installroot)s/config.xml
sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" %(installroot)s/config.xml
slapos/recipe/boinc/template/sed_update.in
View file @ 4bb47083
......@@ -10,7 +10,5 @@ sed -i.old -e "/<\/boinc>/i\<app>\n<name>$1</name>\n<user_friendly_name>$1</user
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_bitwise_validator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_assimilator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" $2/config.xml
sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" $2/config.xml
fi
\ No newline at end of file
software/boinc/software.cfg
View file @ 4bb47083
......@@ -43,7 +43,7 @@ exec-extension =
#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
platform = x86_64-pc-linux-gnu
#Work Unit number number of work unit
wu-number = 4
wu-number = 2
[template-base]
recipe = slapos.recipe.download
......
stack/boinc/buildout.cfg
View file @ 4bb47083
......@@ -41,7 +41,7 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg
md5sum = 003bff525faa1e63913fa5f38c18becd
md5sum = 22de71bad135ab3c760c63e537dcb182
mode = 0644
#Template for deploying MySQL Database Server
......@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
mode = 0644
filename = apache.in
md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
md5sum = 030892494ce87357e6e09dcd89187bf4
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Local development
......
stack/boinc/instance-boinc.cfg
View file @ 4bb47083
......@@ -20,7 +20,9 @@ parts =
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
#temporary solution
newest = false
offline = false
# Creation of all needed directories
......@@ -226,7 +228,7 @@ wu-number = $${slap-parameter:wu-number}
binary = $${slap-parameter:binary}
platform = $${slap-parameter:platform}
extension = $${slap-parameter:extension}
template-result = $${slap-parameter:template-wu}
template-result = $${slap-parameter:template-result}
template-wu = $${slap-parameter:template-wu}
input-file = $${slap-parameter:input-file}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7