Skip to content

slapos
slapos
Commit 730f737e authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Allow to update wu and deploy a fully boinc-app using slapparameter

parent dd314877
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 76 deletions
slapos/recipe/boinc/__init__.py
View file @ 730f737e
......@@ -185,11 +185,15 @@ class Recipe(GenericBaseRecipe):
#Generate Boinc start project wrapper
start_args = [os.path.join(self.installroot, 'bin/start')]
start_boinc = os.path.join(self.home, '.start_boinc')
if os.path.exists(start_boinc):
os.unlink(start_boinc)
boinc_parameter = dict(service_status=service_status,
installroot=self.installroot, drop_install=drop_install,
mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
mysql_user=self.username, mysql_password=self.password,
database=self.database, PATH=environment['PATH'])
database=self.database, PATH=environment['PATH'],
python_path=python_path, start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'),
'%s.configure.restart_boinc' % __name__,
......@@ -208,7 +212,10 @@ class App(GenericBaseRecipe):
def install(self):
if self.options['app-name'].strip() == '' or \
self.options['version'].strip() == '':
#don't deploy empty application...skipped
return []
path_list = []
package = self.options['boinc'].strip()
#Define environment variable here
......@@ -232,12 +239,13 @@ class App(GenericBaseRecipe):
bash = os.path.join(home, 'bin', 'update_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip()))
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip()))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
service_status = os.path.join(home, '.start_service')
start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip()
version = self.options['version'].strip()
platform = self.options['platform'].strip()
......@@ -253,9 +261,8 @@ class App(GenericBaseRecipe):
appname=appname, binary_name=bin_name,
version=version, platform=platform,
application=application, environment=environment,
service_status=service_status,
wu_name=self.options['wu-name'].strip(),
wu_number=self.options['wu-number'].strip(),
start_boinc=start_boinc,
wu_number=int(self.options['wu-number'].strip()),
t_result=self.options['template-result'].strip(),
t_wu=self.options['template-wu'].strip(),
t_input=self.options['input-file'].strip(),
......@@ -270,6 +277,7 @@ class App(GenericBaseRecipe):
return path_list
update = install
class Client(GenericBaseRecipe):
"""Deploy a fully fonctionnal boinc client connected to a boinc server instance"""
......
slapos/recipe/boinc/configure.py
View file @ 730f737e
......@@ -131,32 +131,41 @@ def restart_boinc(args):
os.environ['PATH'] = args['PATH']
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
# startProcess([binstart], env)
# startProcess([binstop], env)
os.system(binstop)
os.system(binstart)
status = open(args['start_boinc'], "w")
status.write("started")
status.close()
print "Done."
def deployApp(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
print "Cheking if needed to install %s..." % args['appname']
if os.path.exists(os.path.join(args['installroot'], "." + args['appname'])):
print args['appname'] + " is already installed in this Boinc instance... skipped"
return
#Sleep until file .start_service exist (Mark the end of boinc configuration)
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
dropapp = False
if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip())
if args['previous_wu'] >= args['wu_number']:
print args['appname'] + " version " + args['version'] + " is already installed in this Boinc instance... skipped"
return
else:
print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number'])
else:
args['previous_wu'] = 0
dropapp = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
while True:
print "Search for file %s..." % args['service_status']
if not os.path.exists(args['service_status']):
print "Search for file %s..." % args['start_boinc']
if not os.path.exists(args['start_boinc']):
print "File not found... sleep for 3 secondes"
time.sleep(3)
else:
break
print "sleeps for 30 seconds while waiting for the end of the execution of boinc_start"
time.sleep(30)
print "setup directories..."
args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname']+'_input')
args['appname'] + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
......@@ -164,30 +173,45 @@ def deployApp(args):
t_wu = os.path.join(args['templates'], args['appname']+'_wu')
if not os.path.exists(base_app):
os.mkdir(base_app)
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
else:
if os.path.exists(t_result):
os.unlink(t_result)
if os.path.exists(t_result):
os.unlink(t_wu)
shutil.copy(args['t_result'], t_result)
shutil.copy(args['t_wu'], t_wu)
if not os.path.exists(args['inputfile']):
os.symlink(args['t_input'], args['inputfile'])
shutil.copy(args['binary'], os.path.join(args['application'],
args['binary_name']))
if dropapp:
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
else:
if os.path.exists(t_result):
os.unlink(t_result)
if os.path.exists(t_result):
os.unlink(t_wu)
shutil.copy(args['t_result'], t_result)
shutil.copy(args['t_wu'], t_wu)
if not os.path.exists(args['inputfile']):
os.symlink(args['t_input'], args['inputfile'])
shutil.copy(args['binary'], os.path.join(args['application'],
args['binary_name']))
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml')
config_xml = os.path.join(args['installroot'], 'config.xml')
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml')
config_xml = os.path.join(args['installroot'], 'config.xml')
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w')
p_sign = subprocess.Popen([sign, os.path.join(args['application'],
args['binary_name']), privateKeyFile], stdout=output,
stderr=subprocess.STDOUT)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
#END if drop-app HERE
print "Running xadd script..."
env = os.environ
......@@ -196,19 +220,6 @@ def deployApp(args):
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w')
p_sign = subprocess.Popen([sign, os.path.join(args['application'],
args['binary_name']), privateKeyFile], stdout=output,
stderr=subprocess.STDOUT)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
print "Running script bin/update_versions..."
updt_version = os.path.join(args['installroot'], 'bin/update_versions')
p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
......@@ -232,20 +243,21 @@ def deployApp(args):
os.system(binstart)
print "Boinc Application deployment is done... writing end signal file..."
sfile = open(os.path.join(args['installroot'], "."+args['appname']), 'w')
sfile.write("done")
sfile = open(token, 'w')
sfile.write(str(args['wu_number']))
sfile.close()
def create_wu(args, env):
count = int(args['wu_number'])
t_result = "templates/" + args['appname'] + '_result'
t_wu = "templates/" + args['appname'] + '_wu'
wu_name = 'wu_' + args['appname'] + args['version']
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', args['wu_name'],
'--wu_template', "templates/"+args['appname'] + '_wu',
'--result_template', "templates/"+args['appname'] + '_result',
'--appname', args['appname'], '--wu_name', wu_name,
'--wu_template', t_wu, '--result_template', t_result,
args['appname']+'_input']
for i in range(count):
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['wu_name']+str(i+1)
launch_args[4] = wu_name + str(i+1)
startProcess(launch_args, env, args['installroot'])
def startProcess(launch_args, env=None, cwd=None):
......
software/boinc/boinc-app.cfg
View file @ 730f737e
......@@ -16,7 +16,10 @@ parts =
frontend-promise
content-promise
publish-connection-informations
#User application
boinc-app
#slap application
slap-application
extends = ${template-boinc:output}
......@@ -30,11 +33,10 @@ app-name = ${boinc-application:app-name}
version = ${boinc-application:version}
platform = ${boinc-application:platform}
extension = ${boinc-application:exec-extension}
dash = ${dash:location}/bin/dash
#templates
template-result = ${template_result:location}/${template_result:filename}
template-wu = ${template_wu:location}/${template_wu:filename}
#Work Unit
wu-name = ${boinc-application:wu-name}
wu-number = ${boinc-application:wu-number}
input-file = ${template_input:location}/${template_input:filename}
software/boinc/software.cfg
View file @ 730f737e
[buildout]
develop =
/srv/slapgrid/slappart19/srv//runner/project/slapos.github
parts =
boinc-instance
template
......@@ -9,8 +12,8 @@ parts =
template_input
slapos-cookbook
instance-egg
slapos.cookbook-repository
check-recipe
# slapos.cookbook-repository
# check-recipe
extends =
../../stack/boinc/buildout.cfg
......@@ -21,7 +24,7 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/boinc-app.cfg
output = ${buildout:directory}/template-app.cfg
mode = 0644
md5sum = c30c21128e6522d0d6578466f5792517
md5sum = ab01f101cc6280ef07ea61a22a1432d0
[template]
recipe = slapos.recipe.template
......@@ -42,9 +45,8 @@ version = 1.0
exec-extension =
#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
platform = x86_64-pc-linux-gnu
#Work Unit: wu-name without blanc space: wu-number number of work unit
wu-name = simpletest
wu-number = 1
#Work Unit number number of work unit
wu-number = 4
[template-base]
recipe = slapos.recipe.download
......
stack/boinc/buildout.cfg
View file @ 730f737e
[buildout]
# Local development
develop =
${:parts-directory}/slapos.cookbook-repository
#develop =
# ${:parts-directory}/slapos.cookbook-repository
parts =
slapos-cookbook
......@@ -41,7 +41,7 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg
md5sum = 2ab17fad66dd07677367fcb7283d55e5
md5sum = 6746390ac3e19a8b1b933e0722084675
mode = 0644
#Template for deploying MySQL Database Server
......
stack/boinc/instance-boinc.cfg
View file @ 730f737e
......@@ -2,6 +2,7 @@
parts =
boinc-server
slap-application
certificate-authority
ca-stunnel
logrotate
......@@ -72,7 +73,7 @@ stunnel-binary = ${stunnel:location}/bin/stunnel
remote-host = $${mariadb-urlparse:host}
remote-port = $${mariadb-urlparse:port}
local-host = $${slap-network-information:local-ipv4}
local-port = 3306
local-port = 33060
log-file = $${basedirectory:log}/stunnel.log
config-file = $${directory:stunnel-conf}/stunnel.conf
key-file = $${directory:stunnel-conf}/stunnel.key
......@@ -124,6 +125,7 @@ output = $${rootdirectory:etc}/apache.conf
document_root = $${rootdirectory:srv}/www/
pid_file = $${basedirectory:run}/apache.pid
lock_file = $${basedirectory:run}/apache.lock
cgid_sock = $${basedirectory:run}/cgid.sock
ip = $${slap-network-information:global-ipv6}
port = 8080
error_log = $${directory:httpd-log}/error.log
......@@ -194,11 +196,13 @@ develop-egg = ${buildout:develop-eggs-directory}
svn-binary = ${subversion:location}/bin
perl-binary = ${perl:location}/bin
python-binary = ${buildout:executable}
dash = ${dash:location}/bin/dash
#Apache and php
ip = $${apache-php:ip}
port = $${apache-php:port}
htpasswd = ${apache:location}/bin/htpasswd
apache-pid = $${httpd-conf:pid_file}
php-ini = $${apache-php:php-ini-dir}/php.ini
php-bin = ${apache-php:location}/bin/php
php-wrapper = $${rootdirectory:bin}php
......@@ -210,6 +214,22 @@ mysql-database = $${mariadb-urlparse:path}
mysql-host = $${stunnel:local-host}
mysql-port = $${stunnel:local-port}
#This allow to deploy or to update Boinc application using only slapparameters from
#vifib BOINC instance parameters
[slap-application]
<= boinc-server
recipe = slapos.cookbook:boinc.app
#appname and version is require to update wu-number
app-name = $${slap-parameter:app-name}
version = $${slap-parameter:version}
wu-number = $${slap-parameter:wu-number}
binary = $${slap-parameter:binary}
platform = $${slap-parameter:platform}
extension = $${slap-parameter:extension}
template-result = $${slap-parameter:template-wu}
template-wu = $${slap-parameter:template-wu}
input-file = $${slap-parameter:input-file}
# Deploy logrotate, cron, configure it
[logrotate]
recipe = slapos.cookbook:logrotate
......@@ -272,7 +292,7 @@ command = $${logrotate:wrapper}
recipe = slapos.cookbook:cron.d
name = cronjob
frequency = 0,5,10,15,20,25,30,35,40,45,50,55 * * * *
command = $${boinc-server:installroot}$${boinc-server:project}/bin/start --cron
command = PATH=$${rootdirectory:bin}:$PATH $${boinc-server:installroot}/bin/start --cron
# Request frontend
......@@ -351,6 +371,16 @@ dbname = boinctest
project = boinc_test
full-name = Boinc Project SAMPLE
copyright-holder = REPLACE WITH COPYRIGHT HOLDER
#This parameter is use to update or to deploy BOINC application
binary =
app-name =
version =
platform =
extension =
template-result =
template-wu =
wu-number =
input-file =
# Default value if no domain is specified
domain =
# Default value if no ssh parameter is specified
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7