Skip to content

slapos
slapos
Commit 6779b9d9 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Move boinc & bonjourgrid to unstable and drop related recipes

parent 6e593795
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1139 deletions
setup.py
View file @ 6779b9d9
......@@ -81,11 +81,6 @@ setup(name=name,
'apachephp = slapos.recipe.apachephp:Recipe',
'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
'apacheproxy = slapos.recipe.apacheproxy:Recipe',
'boinc = slapos.recipe.boinc:Recipe',
'boinc.app = slapos.recipe.boinc:App',
'boinc.client = slapos.recipe.boinc:Client',
'bonjourgrid = slapos.recipe.bonjourgrid:Recipe',
'bonjourgrid.client = slapos.recipe.bonjourgrid:Client',
'certificate_authority = slapos.recipe.certificate_authority:Recipe',
'certificate_authority.request = slapos.recipe.certificate_authority:Request',
'check_page_content = slapos.recipe.check_page_content:Recipe',
......
slapos/recipe/boinc/__init__.py deleted 100644 → 0
View file @ 6e593795
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import pwd
import json
import signal
import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational boinc architecture."""
def __init__(self, buildout, name, options):
#get current slapuser name
stat_info = os.stat(options['home'].strip())
options['user'] = pwd.getpwuid(stat_info.st_uid)[0]
url_base = options['url-base']
project = options['project'].strip()
root = options['installroot'].strip()
options['home_page'] = url_base + "/" + project
options['admin_page'] = url_base + "/" + project + "_ops/"
options['result_page'] = url_base + "/" + project + "_result/"
options['cronjob'] = os.path.join(root, project+'.cronjob')
return GenericBaseRecipe.__init__(self, buildout, name, options)
def _options(self, options):
#Path of boinc compiled package
self.package = options['boinc'].strip()
self.sourcedir = options['source'].strip()
self.home = options['home'].strip()
self.project = options['project'].strip()
self.fullname = options['fullname'].strip()
self.copyright = options['copyright'].strip()
self.installroot = options['installroot'].strip()
self.boinc_egg = os.path.join(self.package, 'lib/python2.7/site-packages')
self.developegg = options['develop-egg'].strip()
self.wrapperdir = options['wrapper-dir'].strip()
self.passwd = options['passwd'].strip()
#Get binary path
self.svn = options['svn-binary'].strip()
self.perl = options['perl-binary'].strip()
self.pythonbin = options['python-binary'].strip()
#Apache php informations
self.url_base =options['url-base'].strip()
self.htpasswd = options['htpasswd'].strip()
self.phpini = options['php-ini'].strip()
self.phpbin = options['php-bin'].strip()
#get Mysql parameters
self.username = options['mysql-username'].strip()
self.password = options['mysql-password'].strip()
self.database = options['mysql-database'].strip()
self.mysqlhost = options['mysql-host'].strip()
self.mysqlport = options['mysql-port'].strip()
def haschanges(self):
config_file = os.path.join(self.home, '.config')
current = [self.fullname, self.copyright,
self.password, self.mysqlhost, self.installroot,
self.project, self.passwd, self.url_base]
previous = []
result = False
if os.path.exists(config_file):
previous = open(config_file, 'r').read().split('#')
#Check if config has changed
if len(current) != len(set(current).intersection(set(previous))) or \
not os.path.exists(self.installroot) or \
not os.path.exists(os.path.join(self.home, '.start_service')):
result = True
open(config_file, 'w').write('#'.join(current))
return result
def install(self):
path_list = []
make_project = os.path.join(self.package, 'bin/make_project')
niceprojectname = self.project + "@Home"
slapuser = self.options['user']
#Check if given URL is not empty (case of URL request with frontend)
if not self.url_base:
raise Exception("URL_BASE is still empty. Can not use it")
#Define environment variable here
python = os.path.join(self.home, 'bin/python')
python_path = self.boinc_egg
if not os.path.exists(python):
os.symlink(self.pythonbin, python)
for f in os.listdir(self.developegg):
dir = os.path.join(self.developegg, f)
if os.path.isdir(dir):
python_path += ":" + dir
bin_dir = os.path.join(self.home, 'bin')
environment = dict(
PATH=os.pathsep.join([self.svn, bin_dir, self.perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#Generate wrapper for php
wrapperphp = os.path.join(self.home, 'bin/php')
php_wrapper = self.createWrapper(wrapperphp,
(self.phpbin, '-c', self.phpini),
)
path_list.append(php_wrapper)
mysql_dict = dict(db=self.database,
host=self.mysqlhost, port=self.mysqlport,
user=self.username, passwd=self.password)
#Generate python script for MySQL database test (starting)
file_status = os.path.join(self.home, '.boinc_config')
if os.path.exists(file_status):
os.unlink(file_status)
mysql_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_config'),
'%s.configure.checkMysql' % __name__,
(environment, mysql_dict, file_status)
)
# Generate make project wrapper file
readme_file = os.path.join(self.installroot, self.project+'.readme')
launch_args = [make_project, '--url_base', self.url_base, "--db_name",
self.database, "--db_user", self.username, "--db_passwd",
self.password, "--project_root", self.installroot, "--db_host",
self.mysqlhost, "--user_name", slapuser, "--srcdir",
self.sourcedir, "--no_query"]
drop_install = self.haschanges()
request_make_boinc = os.path.join(self.home, '.make_project')
if drop_install:
#Allow to restart Boinc installation from the begining
launch_args += ["--delete_prev_inst", "--drop_db_first"]
open(request_make_boinc, 'w').write('Make Requested')
if os.path.exists(readme_file):
os.unlink(readme_file)
launch_args += [self.project, niceprojectname]
install_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'make_project'),
'%s.configure.makeProject' % __name__,
(file_status, launch_args, request_make_boinc, environment)
)
path_list.append(install_wrapper)
#generate sh script for project configuration
bash = os.path.join(self.home, 'bin', 'project_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('project_config.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
user=slapuser, fullname=self.fullname,
copyright=self.copyright, installroot=self.installroot))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#After make_project run configure_script to perform and restart apache php services
service_status = os.path.join(self.home, '.start_service')
parameter = dict(
readme=readme_file,
htpasswd=self.htpasswd,
installroot=self.installroot,
username=slapuser,
passwd=self.passwd,
xadd=os.path.join(self.installroot, 'bin/xadd'),
environment=environment,
service_status=service_status,
drop_install=drop_install,
sedconfig=bash
)
start_service = self.createPythonScript(
os.path.join(self.wrapperdir, 'config_project'),
'%s.configure.services' % __name__, (parameter,)
)
path_list.append(start_service)
#Generate Boinc start project wrapper
start_args = [os.path.join(self.installroot, 'bin/start')]
start_boinc = os.path.join(self.home, '.start_boinc')
if os.path.exists(start_boinc):
os.unlink(start_boinc)
boinc_parameter = dict(service_status=service_status,
installroot=self.installroot, drop_install=drop_install,
mysql_dict=mysql_dict, environment=environment,
start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'),
'%s.configure.restart_boinc' % __name__,
(boinc_parameter,)
)
path_list.append(start_wrapper)
return path_list
update = install
class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self, app):
"""This is used to download app files if necessary and update options values"""
for key in ('input-file', 'template-result', 'template-wu', 'binary'):
param = app[key]
if param and (param.startswith('http') or param.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'tmp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(param, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
os.chmod(path, mode)
app[key] = path
def getAppList(self):
"""Load parameters,
check if parameter send is valid to install or update application"""
app_list = json.loads(self.options['boinc-app-list'])
if not app_list:
return None
default_template_result = self.options.get('default-template-result', '').strip()
default_template_wu = self.options.get('default-template-wu', '').strip()
default_extension = self.options.get('default-extension', '').strip()
default_platform = self.options.get('default-platform', '').strip()
for app in app_list:
for version in app_list[app]:
current_app = app_list[app][version]
#Use default value if empty and Use_default is True
#Initialize all values to empty if not define by the user
if current_app['use_default']:
current_app['template-result'] = current_app.get('template-result',
default_template_result).strip()
current_app['template-wu'] = current_app.get('template-wu',
default_template_wu).strip()
current_app['extension'] = current_app.get('extension',
default_extension).strip()
current_app['platform'] = current_app.get('platform',
default_platform).strip()
else:
current_app['template-result'] = current_app.get('template-result', '').strip()
current_app['template-wu'] = current_app.get('template-wu', '').strip()
current_app['extension'] = current_app.get('extension', '').strip()
current_app['platform'] = current_app.get('platform', '').strip()
current_app['input-file'] = current_app.get('input-file', '').strip()
current_app['wu-number'] = current_app.get('wu-number', 1)
#for new application, check if parameter is complete
appdir = os.path.join(self.options['installroot'].strip(), 'apps',
app, version)
if not os.path.exists(appdir):
if not current_app['template-result'] or not current_app['binary'] \
or not current_app['input-file'] or not current_app['template-wu'] \
or not current_app['platform']:
print "BOINC-APP: ERROR - Invalid argements values for % ...operation cancelled" % app
app_list[app][version] = None
continue
#write application to install
request_file = os.path.join(self.options['home'].strip(),
'.install_' + app + version)
toInstall = open(request_file, 'w')
toInstall.write('install or update')
toInstall.close()
return app_list
def install(self):
app_list = self.getAppList()
path_list = []
package = self.options['boinc'].strip()
#Define environment variable here
developegg = self.options['develop-egg'].strip()
python_path = os.path.join(package, 'lib/python2.7/site-packages')
home = self.options['home'].strip()
user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip()
svn = self.options['svn-binary'].strip()
for f in os.listdir(developegg):
dir = os.path.join(developegg, f)
if os.path.isdir(dir):
python_path += ":" + dir
bin_dir = os.path.join(home, 'bin')
environment = dict(
PATH=os.pathsep.join([svn, bin_dir, perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#generate project.xml and config.xml script updater
bash = os.path.join(home, 'bin', 'update_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip()))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#If useful, download necessary files and update options path
start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip()
apps_dir = os.path.join(installroot, 'apps')
wrapperdir = self.options['wrapper-dir'].strip()
project = self.options['project'].strip()
lockfile = os.path.join(self.options['home'].strip(), 'app_install.lock')
fd = os.open(lockfile, os.O_RDWR|os.O_CREAT)
os.close( fd )
for appname in app_list:
for version in app_list[appname]:
if not app_list[appname][version]:
continue
self.downloadFiles(app_list[appname][version])
platform = app_list[appname][version]['platform']
application = os.path.join(apps_dir, appname, version, platform)
if app_list[appname][version]['binary'] and not platform:
print "BOINC-APP: WARNING - Cannot specify binary without giving platform value"
app_list[appname][version]['binary'] = '' #Binary will not be updated
parameter = dict(installroot=installroot,
appname=appname, project=project,
version=version, platform=platform,
application=application, environment=environment,
start_boinc=start_boinc,
wu_number=app_list[appname][version]['wu-number'],
t_result=app_list[appname][version]['template-result'],
t_wu=app_list[appname][version]['template-wu'],
t_input=app_list[appname][version]['input-file'],
binary=app_list[appname][version]['binary'],
extension=app_list[appname][version]['extension'],
bash=bash, home_dir=home,
lockfile=lockfile,
)
deploy_app = self.createPythonScript(
os.path.join(wrapperdir, 'boinc_%s' % appname),
'%s.configure.deployApp' % __name__, (parameter,)
)
path_list.append(deploy_app)
return path_list
update = install
class Client(GenericBaseRecipe):
"""Deploy a fully fonctionnal boinc client connected to a boinc server instance"""
def __init__(self, buildout, name, options):
#get current uig to create a unique rpc-port for this client
stat_info = os.stat(options['home'].strip())
options['rpc-port'] = pwd.getpwuid(stat_info.st_uid)[2] + 5000
return GenericBaseRecipe.__init__(self, buildout, name, options)
def install(self):
path_list = []
boincbin = self.options['boinc-bin'].strip()
cmdbin = self.options['cmd-bin'].strip()
installdir = self.options['install-dir'].strip()
url = self.options['server-url'].strip()
key = self.options['key'].strip()
boinc_wrapper = self.options['client-wrapper'].strip()
cmd_wrapper = self.options['cmd-wrapper'].strip()
remote_host = os.path.join(installdir, 'remote_hosts.cfg')
open(remote_host, 'w').write(self.options['ip'].strip())
#Generate wrapper for boinc cmd
base_cmd = [cmdbin, '--host', str(self.options['rpc-port']),
'--passwd', self.options['passwd'].strip()]
cc_cmd = ''
if self.options['cconfig'].strip() != '':
config_dest = os.path.join(installdir, 'cc_config.xml')
file = open(config_dest, 'w')
file.write(open(self.options['cconfig'].strip(), 'r').read())
file.close()
cc_cmd = '--read_cc_config'
cmd = self.createPythonScript(cmd_wrapper,
'%s.configure.runCmd' % __name__,
(base_cmd, cc_cmd, installdir, url, key)
)
path_list.append(cmd)
#Generate BOINC client wrapper
boinc = self.createWrapper(boinc_wrapper,
(boincbin, '--allow_multiple_clients', '--gui_rpc_port',
str(self.options['rpc-port']), '--allow_remote_gui_rpc',
'--dir', installdir, '--redirectio', '--check_all_logins'),
)
path_list.append(boinc)
return path_list
slapos/recipe/boinc/configure.py deleted 100644 → 0
View file @ 6e593795
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import sys
import subprocess
import time
import shutil
import re
import filecmp
from lock_file import LockFile
def checkMysql(environment, connect_kw, file_status=None):
sys.path += environment['PYTHONPATH'].split(':')
import MySQLdb
#Sleep until mysql server becomes available
while True:
try:
MySQLdb.connect(**connect_kw).close()
break
except Exception, ex:
print "The result is: \n" + ex.message
print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2)
print "Successfully connect to MySQL database... "
if file_status:
writeFile(file_status, "starting")
def checkFile(file, stime):
"""Loop until 'file' is created (exist)"""
while True:
print "Search for file %s..." % file
if not os.path.exists(file):
print "File not found... sleep for %s secondes" % stime
time.sleep(stime)
else:
break
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
environment = args['environment']
if args['drop_install']:
checkFile(args['service_status'], 3)
else:
checkMysql(environment, args['mysql_dict'], args.get('file_status'))
print "Restart Boinc..."
env = os.environ.copy()
env.update(environment)
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
writeFile(args['start_boinc'], "started")
print "Done."
def check_installRequest(args):
print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
return False
os.unlink(install_request_file)
return True
def copy_file(source, dest):
""""Copy file with source to dest with auto replace
return True if file has been copied and dest ha been replaced
"""
result = False
if source and os.path.exists(source):
if os.path.exists(dest):
if filecmp.cmp(dest, source):
return False
os.unlink(dest)
result = True
shutil.copy(source, dest)
return result
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute executable.\nThe error was: %s" % result
return False
return True
def makeProject(make_sig, launch_args, request_file, extra_environ):
"""Run BOINC make_project script but once only"""
#Wait for DateBase initialization...
checkFile(make_sig, 3)
print "Cheking if needed to run BOINC make_project..."
if os.path.exists(request_file):
env = os.environ.copy()
env.update(extra_environ)
if startProcess(launch_args, env=env):
os.unlink(request_file)
print "Finished running BOINC make_projet...Ending"
else:
print "No new request for make_project. Exiting..."
def services(args):
"""This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..."
if not args['drop_install']:
print "Not need to install Boinc-server...skipped"
return
#Sleep until file 'boinc_project'.readme exist
checkFile(args['readme'], 3)
topath = os.path.join(args['installroot'], 'html/ops/.htpasswd')
print "Generating .htpasswd file... File=%s" % topath
passwd = open(args['passwd'], 'r').read()
htpwd_args = [args['htpasswd'], '-b', '-c', topath, args['username'], passwd]
if not startProcess(htpwd_args):
return
print "execute script xadd..."
env = os.environ.copy()
env.update(args['environment'])
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Update files and directories permissions..."
upload = os.path.join(args['installroot'], 'upload')
inc = os.path.join(args['installroot'], 'html/inc')
languages = os.path.join(args['installroot'], 'html/languages')
compiled = os.path.join(args['installroot'], 'html/languages/compiled')
user_profile = os.path.join(args['installroot'], 'html/user_profile')
forum_file = os.path.join(args['installroot'], 'html/ops/create_forums.php')
project_inc = os.path.join(args['installroot'], 'html/project/project.inc')
cmd = "chmod 02700 -R %s %s, %s %s %s" % (upload, inc,
languages, compiled, user_profile)
os.system("chmod g+w -R " + args['installroot'])
os.system(cmd)
os.system("chmod 700 %s" % os.path.join(args['installroot'], 'keys'))
os.system("chmod o+x " + inc)
os.system("chmod -R o+r " + inc)
os.system("chmod o+x " + languages)
os.system("chmod o+x " + compiled)
sed_args = [args['sedconfig']]
startProcess(sed_args)
#Execute php create_forum.php...
print "Boinc Forum: Execute php create_forum.php..."
cwd = os.path.join(args['installroot'], 'html/ops')
if not startProcess(["php", forum_file], env, cwd):
return
writeFile(args['service_status'], "started")
def deployApp(args):
"""Deploy Boinc App with lock"""
print "Asking to enter in execution with lock mode..."
with LockFile(args['lockfile'], wait=True):
print "acquire the lock file..."
deployManagement(args)
print "Exit execution with lock..."
def deployManagement(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
if not check_installRequest(args):
return
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
newInstall = False
if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip())
if args['previous_wu'] < args['wu_number']:
print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number'])
else:
args['previous_wu'] = 0
newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
env = os.environ.copy()
env.update(args['environment'])
print "setup directories..."
numversion = args['version'].replace('.', '')
args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname'] + numversion + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
t_result = os.path.join(args['templates'],
args['appname'] + numversion + '_result')
t_wu = os.path.join(args['templates'],
args['appname'] + numversion + '_wu')
binary_name = args['appname'] +"_"+ args['version'] +"_"+ \
args['platform'] + args['extension']
binary = os.path.join(args['application'], binary_name)
signBin = False
if not os.path.exists(base_app):
os.mkdir(base_app)
if newInstall:
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
copy_file(args['t_result'], t_result)
copy_file(args['t_wu'], t_wu)
signBin = copy_file(args['binary'], binary)
if args['t_input']:
if os.path.exists(args['inputfile']):
os.unlink(args['inputfile'])
os.symlink(args['t_input'], args['inputfile'])
project_xml = os.path.join(args['installroot'], 'project.xml')
findapp = re.search("<name>(%s)</name>" % args['appname'],
open(project_xml, 'r').read())
if not findapp:
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
if signBin:
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT, env=env)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
print "execute script xadd..."
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Running script bin/update_versions..."
updt_version = os.path.join(args['installroot'], 'bin/update_versions')
p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env,
cwd=args['installroot'])
p_version.stdin.write('y\ny\n')
result = p_version.communicate()[0]
p_version.stdin.close()
if p_version.returncode is None or p_version.returncode != 0:
print "Failed to execute bin/update_versions.\nThe error was: %s" % result
return
print "Fill the database... calling bin/create_work..."
create_wu(args, env)
print "Restart Boinc..."
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number']))
def create_wu(args, env):
"""Create or update number of work unit for an existing boinc application"""
numversion = args['version'].replace('.', '')
t_result = "templates/" + args['appname'] + numversion + '_result'
t_wu = "templates/" + args['appname'] + numversion + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1',
args['appname'] + numversion + '_input']
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + numversion + '_nodelete'
startProcess(launch_args, env, args['installroot'])
def runCmd(base_cmd, cc_cmd, installdir, url, key):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(installdir, 'client_state.xml')
checkFile(client_config, 5)
time.sleep(10)
#Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
base_cmd[2] = host + ':' + base_cmd[2]
print "Run boinccmd with host at %s " % base_cmd[2]
project_args = base_cmd + ['--project_attach', url, key]
startProcess(project_args, cwd=installdir)
if cc_cmd:
#Load or reload cc_config file
startProcess(base_cmd + [cc_cmd], cwd=installdir)
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
f.close()
slapos/recipe/boinc/template/project_config.in deleted 100644 → 0
View file @ 6e593795
#!%(dash)s
sed -i "/remove the die/d" %(installroot)s/html/ops/create_forums.php
sed -i "s#REPLACE WITH PROJECT NAME#%(fullname)s#" %(installroot)s/html/project/project.inc
sed -i "s#REPLACE WITH COPYRIGHT HOLDER#%(copyright)s#" %(installroot)s/html/project/project.inc
sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" %(installroot)s/config.xml
sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" %(installroot)s/config.xml
slapos/recipe/boinc/template/sed_update.in deleted 100644 → 0
View file @ 6e593795
#!%(dash)s
if [ $# -ne 2 ]
then
echo not enough argument.
echo ARGS: appname installroot
else
sed -i.old -e "/<\/boinc>/i\<app>\n<name>$1</name>\n<user_friendly_name>$1</user_friendly_name>\n</app>" $2/project.xml
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_bitwise_validator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_assimilator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
fi
\ No newline at end of file
slapos/recipe/bonjourgrid/__init__.py deleted 100644 → 0
View file @ 6e593795
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import pwd
import signal
import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational BonjourGrid Master architecture."""
def install(self):
path_list = []
condor_wrapper = self.options['condor-wrapper'].strip()
boinc_wrapper = self.options['boinc-wrapper'].strip()
startCondor = startBoinc = False
condor_wrapper_list = []
boinc_wrapper_list = []
for file in os.listdir(condor_wrapper):
condor_wrapper_list.append(os.path.join(condor_wrapper, file))
for file in os.listdir(boinc_wrapper):
boinc_wrapper_list.append(os.path.join(boinc_wrapper, file))
type = self.options['type'].strip()
if type == "condor":
startCondor = True
project = self.options['condor_host'].strip()
if type == "boinc":
project = self.options['url-boinc'].strip()
startBoinc = True
#Generate BOINC/Condor launcher script
grid_wrapper = self.options['boinc_condor_wrapper'].strip()
parameters = dict(startCondor=startCondor, startBoinc=startBoinc,
bg_base=self.options['work_dir'].strip(),
condor_wrapper_list=condor_wrapper_list,
boinc_wrapper_list=boinc_wrapper_list)
bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
__name__ + '.configure.launchScript',
(parameters,)
)
path_list.append(bonjourGrid_wrapper)
#Generate wrapper for BonjourGrid Master
bonjourgrid_master = self.options['master_script'].strip()
python = self.options['python-bin'].strip()
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_master, '--log_file', log,
'--pid_file', pid_file,
'--master_wrapper', grid_wrapper,
'--directory', self.options['work_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
'--num_workers', self.options['nworkers'].strip(),
),
)
path_list.append(wrapper)
#generate Computer information file
config_info_file = os.path.join(self.options['work_dir'].strip(),
'machineinfo.sh')
config_info = self.createFile(config_info_file,
self.substituteTemplate(self.getTemplateFilename('machineinfo.sh.in'),
dict(ip_address=self.options['ipv6'].strip(),
project=project,
middleware=type)))
os.chmod(config_info_file, 0o744)
path_list.append(config_info)
update = install
class Client(GenericBaseRecipe):
def install(self):
path_list = []
boinc_script = self.options['boinc_script'].strip()
condor_script = self.options['condor_script'].strip()
#Generate wrapper for BonjourGrid Worker
bonjourgrid_client = self.options['client_script'].strip()
python = self.options['python-bin'].strip()
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_client, '--log_file', log,
'--pid_file', pid_file,
'--boinc_wrapper', boinc_script,
'--condor_wrapper', condor_script,
'--directory', self.options['work_dir'].strip(),
'--install_directory', self.options['install_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
),
)
path_list.append(wrapper)
#generate BOINC and Condor configure script for bonjourgrid
boinc_wrapper = self.createPythonScript(boinc_script,
'%s.boinc.runBoinc' % __name__,
dict(ipv6=self.options['ipv6'].strip(),
email=self.options['email'].strip(),
boinc_wrapper=self.options['boinc_wrapper'].strip(),
boinc_cmd=self.options['boinc_cmd'].strip(),
boinc_rpc_port=self.options['boinc_rpc_port'],
boinc_install_dir=self.options['boinc_install_dir'].strip(),
boinc_passwd=self.options['boinc_passwd'].strip(),
account_name=self.options['account_name'].strip(),
account_passwd=self.options['account_passwd'].strip(),
)
)
path_list.append(boinc_wrapper)
condor_wrapper = self.createPythonScript(condor_script,
'%s.condor.runCondor' % __name__,
dict(ipv6=self.options['ipv6'].strip(),
condor_bin=self.options['condor_bin'].strip(),
condor_sbin=self.options['condor_sbin'].strip(),
condor_wrapper=self.options['condor_wrapper'].strip(),
condor_config=self.options['condor_config'].strip(),
condor_config_local=self.options['condor_config_local'].strip(),
)
)
path_list.append(condor_wrapper)
slapos/recipe/bonjourgrid/boinc.py deleted 100644 → 0
View file @ 6e593795
# -*- coding: utf-8 -*-
import os
import sys
import re
import subprocess
import time
import signal
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
raise NameError("Failed to execute executable.\nThe error was: %s" % result)
def joinProject(args, base_cmd):
"""Finish BOINC Client configuration with create account and attach project"""
project_args = base_cmd + ['--project_attach', args['project_url'],
args['key']]
startProcess(project_args, cwd=args['boinc_install_dir'])
def createAccount(config, base_cmd):
"""Connect to BOINC Master and create an account
"""
account_args = base_cmd + ['--create_account', config['project_url'],
config['email'], config['account_passwd'],
config['account_name']]
startProcess(account_args, cwd=config['boinc_install_dir'])
account_file = os.path.join(config['boinc_install_dir'], 'create_account.xml')
key = re.search("<authenticator>([\w\d\._]+)</authenticator>",
open(account_file, 'r').read()).group(1)
return key
def runBoinc(config):
if len(sys.argv) < 2:
print "Argument Error: uses %s project_url" % sys.argv[0]
exit(1)
if type(config) == type(""):
print "Error: bonjourgrid.xml parsing error, file not exist or corrupted"
exit(1)
#XXX Using define values here for Boinc Master URL
config['project_url'] = sys.argv[1]
#launch Boinc Client
boinc = subprocess.Popen([config['boinc_wrapper']],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
#Retrieve ipv4 using by Boinc-client in client-configuration
client_config = os.path.join(config['boinc_install_dir'], 'client_state.xml')
while not os.path.exists(client_config):
time.sleep(5)
print "Search for file '%r'..." % client_config
time.sleep(10)
try:
#Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
base_cmd = [config['boinc_cmd'], '--host',
host + ":" + str(config['boinc_rpc_port']),
'--passwd', config['boinc_passwd']]
#Create Account for current instance on BOINC master
print "Create account for current client..."
key = createAccount(config, base_cmd)
config['key'] = key
print "Done. The account key is %s" % key
#Attach project to Boinc Master
print "Attach client to Boinc Master at %s " % config['project_url']
try:
joinProject(config, base_cmd)
except Exception, e:
print e
print "Done! Waiting for Boinc Client now..."
except Exception, e:
#An error occure!!!
os.kill(boinc.pid, signal.SIGTERM)
print e
#wait for Boinc client execution
boinc.wait()
slapos/recipe/bonjourgrid/condor.py deleted 100644 → 0
View file @ 6e593795
# -*- coding: utf-8 -*-
import os
import sys
import re
import subprocess
import time
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
f.close()
def updateCondorConfig(path, path_local, hostname, ipv6):
"""Replace Static information into condor_config and condor_config.local files"""
#replace condor host into condor_config
text = open(path, 'r').read()
text = re.sub(r'\[%s\]' % ipv6, hostname, text, flags=re.IGNORECASE)
writeFile(path, text)
#replace condor host into condor_config.local
text = open(path_local, 'r').read()
text = re.sub(r'\[%s\]' % ipv6, hostname, text, flags=re.IGNORECASE)
writeFile(path_local, text)
def updateCondorWrapper(folder, hostname, ipv6):
"""Replace slapos generated value by the true value"""
for file in os.listdir(folder):
path = os.path.join(folder, file)
if os.path.exists(path) and not os.path.isdir(path):
text = re.sub(r'\[%s\]' % ipv6, hostname, open(path, 'r').read(),
flags=re.IGNORECASE)
writeFile(path, text)
def runCondor(config):
if len(sys.argv) < 2:
print "Argument Error: uses %s hostname" % sys.argv[0]
exit(1)
hostname = sys.argv[1]
updateCondorConfig(config['condor_config'], config['condor_config_local'],
hostname, config['ipv6'])
updateCondorWrapper(config['condor_bin'], hostname, config['ipv6'])
updateCondorWrapper(config['condor_sbin'], hostname, config['ipv6'])
#launch Boinc Client
condor = subprocess.Popen([config['condor_wrapper']],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
condor.wait()
slapos/recipe/bonjourgrid/configure.py deleted 100644 → 0
View file @ 6e593795
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import subprocess
import time
def runProcess(args, file):
filename = file.split('/')[-1:][0]
stdout_file = open(os.path.join(args['bg_log'], filename + ".log"), 'w')
process = subprocess.Popen(file, stdout=stdout_file,
stderr=subprocess.STDOUT)
fp = open(os.path.join(args['bg_pid'], filename + '.pid'), 'w')
fp.write(str(process.pid))
fp.close()
return process.pid
def launchScript(args):
print "Sleep for a few second..."
time.sleep(10)
pid_list = []
bg_pid = os.path.join(args['bg_base'], 'pid')
bg_log = os.path.join(args['bg_base'], 'log')
args['bg_pid'] = bg_pid
args['bg_log'] = bg_log
if not os.path.exists(bg_pid):
os.mkdir(bg_pid)
if not os.path.exists(bg_log):
os.mkdir(bg_log)
#launch all Condor wrapper
if args['startCondor']:
for file in args['condor_wrapper_list']:
pid_list.append(runProcess(args, file))
#Launch all BOINC wrapper
if args['startBoinc']:
for file in args['boinc_wrapper_list']:
pid_list.append(runProcess(args, file))
for pid in pid_list:
print "Parent waiting for process child: %s " % pid
result = os.waitpid(pid, 0)
print "Done...", result
slapos/recipe/bonjourgrid/template/machineinfo.sh.in deleted 100755 → 0
View file @ 6e593795
# This script generates a file (with the name `hostname`) of the characteristics of the machine
Mhz=`grep -i Mhz /proc/cpuinfo | cut -d : -f2 | head -1 | cut -f2 -d " "`
mem=`grep -i MemTotal /proc/meminfo | cut -f2 -d :`
Ram=`echo $mem | cut -f1 -d " "`
Cpu=`uname -a | cut -f3 -d " "`
H=`hostname`
Nbproc=`grep processor /proc/cpuinfo |wc -l`
Ip=%(ip_address)s
status="in execution"
secours="host1"
middleware="%(middleware)s"
project="%(project)s"
c="{'"MHZ"':\"$Mhz\",'"Ram"':\"$Ram\",'"Cpu"':\"$Cpu\",'"HOST"':\"$H\",'"IP"':\"$Ip\",'"STATUS"':\"$status\",'"SECOURS"':\"$secours\",'"MIDDLEWARE"':\"$middleware\",'"PROJECT"':\"$project\",}"
filename=`hostname`
echo $c > $filename
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7