Commit 092c7c42 authored by Alain Takoudjou's avatar Alain Takoudjou

Use Json parameters in condor recipe

parent 1601b8fa
......@@ -32,6 +32,7 @@ import filecmp
import urlparse
import shutil
import re
import json
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational condor architecture."""
......@@ -219,67 +220,69 @@ class AppSubmit(GenericBaseRecipe):
def getFiles(self):
"""This is used to download app files if necessary and update options values"""
self.options['file-number'] = 0
if self.options['files']:
files_list = self.options['files'].splitlines()
files_list = [f for f in files_list if f] #remove empty elements
self.options['file-number'] = len(files_list)
for i in range(self.options['file-number']):
value = files_list[i].strip()
pos = str(i)
if value and (value.startswith('http') or value.startswith('ftp')):
self.options['name_'+pos] = os.path.basename(urlparse.urlparse(value)[2])
self.options['file_'+pos] = self.download(value)
os.chmod(self.options['file_'+pos], 0600)
else:
self.options['file_'+pos] = value
executable = self.options['executable']
if executable and (executable.startswith('http') or executable.startswith('ftp')):
self.options['executable'] = self.download(executable,
self.options['executable-name'].strip())
os.chmod(self.options['executable'], 0700)
submit_file = self.options['description-file']
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
self.options['description-file'] = self.download(submit_file, 'submit')
os.chmod(self.options['description-file'], 0600)
app_list = json.loads(self.options['condor-app-list'])
if not app_list:
return None
for app in app_list:
if app_list[app].get('files', None):
file_list = app_list[app]['files']
for file in file_list:
if file and (file.startswith('http') or file.startswith('ftp')):
file_list[file] = self.download(file_list[file])
os.chmod(file_list[file], 0600)
else:
app_list[app]['files'] = {}
executable = app_list[app].get('executable', '')
if executable and (executable.startswith('http') or executable.startswith('ftp')):
app_list[app]['executable'] = self.download(executable,
app_list[app]['executable-name'])
os.chmod(app_list[app]['executable-name'], 0700)
submit_file = app_list[app].get('description-file', '')
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
app_list[app]['description-file'] = self.download(submit_file, 'submit')
os.chmod(app_list[app]['description-file'], 0600)
return app_list
def install(self):
path_list = []
#check if curent condor instance is an condor master
if self.options['machine-role'].strip() != "manager":
print "ERROR: cannot submit a job to Condor worker instance"
return []
raise Exception("Cannot submit a job to Condor worker instance")
#Setup directory
jobdir = self.options['job-dir'].strip()
appdir = os.path.join(jobdir, self.options['app-name'].strip())
submitfile = os.path.join(appdir, 'submit')
appname = self.options['app-name'].strip()
if not os.path.exists(jobdir):
os.mkdir(jobdir)
if not os.path.exists(appdir):
os.mkdir(appdir)
self.getFiles()
self.copy_file(self.options['executable'],
os.path.join(appdir, self.options['executable-name'].strip())
)
install = self.copy_file(self.options['description-file'], submitfile)
sig_install = os.path.join(appdir, '.install')
if install:
with open(sig_install, 'w') as f:
f.write('to_install')
for i in range(self.options['file-number']):
destination = os.path.join(appdir, self.options['name_'+str(i)])
if os.path.exists(destination):
os.unlink(destination)
os.symlink(self.options['file_'+str(i)], destination)
#generate wrapper for submitting job
condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
parameter = dict(submit=condor_submit, sig_install=sig_install,
submit_file='submit',
appname=appname, appdir=appdir)
submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
'%s.configure.submitJob' % __name__, parameter
)
path_list.append(submit_job)
app_list = self.getFiles()
for appname in app_list:
appdir = os.path.join(jobdir, appname)
if not os.path.exists(appdir):
os.mkdir(appdir)
submitfile = os.path.join(appdir, 'submit')
self.copy_file(app_list[appname]['executable'],
os.path.join(appdir, app_list[appname]['executable-name'])
)
install = self.copy_file(app_list[appname]['description-file'], submitfile)
sig_install = os.path.join(appdir, '.install')
if install:
with open(sig_install, 'w') as f:
f.write('to_install')
for file in app_list[appname]['files']:
destination = os.path.join(appdir, file)
if os.path.exists(destination):
os.unlink(destination)
os.symlink(app_list[appname]['files'][file], destination)
#generate wrapper for submitting job
condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
parameter = dict(submit=condor_submit, sig_install=sig_install,
submit_file='submit',
appname=appname, appdir=appdir)
submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
'%s.configure.submitJob' % __name__, parameter
)
path_list.append(submit_job)
return path_list
\ No newline at end of file
......@@ -24,7 +24,7 @@ srv = $${buildout:directory}/srv/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
scripts = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
......@@ -35,7 +35,7 @@ wrapper = $${basedirectory:run}/condor
[condor]
recipe = slapos.cookbook:condor
wrapper-dir = $${basedirectory:services}
wrapper-dir = $${basedirectory:scripts}
ip = $${slap-network-information:global-ipv6}
package = ${condor:location}
rootdirectory = $${buildout:directory}
......@@ -50,46 +50,36 @@ environment =
LD_LIBRARY_PATH=${libexpat:location}/lib:${kerberos:location}/lib:${openldap:location}/lib:${zlib:location}/lib
PATH=${perl:location}/bin:${java:location}/bin:${kerberos:location}/bin:${openldap:location}/bin:%(PATH)s
HOME=$${rootdirectory:condor}
HOSTNAME=$${slap-parameter:condor_host}
HOSTNAME=$${slap-parameter:condor-host}
#Condor user parameter
condor_host = $${slap-parameter:condor_host}
collector_name = $${slap-parameter:collector_name}
condor_host = $${slap-parameter:condor-host}
collector_name = $${slap-parameter:condor-collector-name}
#Condor machine role: worker=submit,execute manager=manager,submit
machine-role = $${slap-parameter:role}
disk-space = $${slap-parameter:diskspace}
allowed-write = $${slap-parameter:allowed-write}
admin-email = $${slap-parameter:admin-email}
machine-role = $${slap-parameter:condor-role}
disk-space = $${slap-parameter:condor-diskspace}
allowed-write = $${slap-parameter:condor-allowed-write}
admin-email = $${slap-parameter:condor-admin-email}
[app-submit]
<= condor
recipe = slapos.cookbook:condor.submit
app-name = $${slap-parameter:app-name}
#user can specifie a list files to use in application, one per line
files = $${slap-parameter:files}
description-file = $${slap-parameter:description-file}
executable = $${slap-parameter:executable}
executable-name = $${slap-parameter:executable-name}
condor-app-list = $${slap-parameter:condor-app-list}
# Publish all instance parameters (url of instance)
[publish-connection-informations]
recipe = slapos.cookbook:publish
ipv6_address = $${condor:ip}
instance_type = $${condor:machine-role}
condor_ipv6_address = $${condor:ip}
condor_instance_type = $${condor:machine-role}
condor_host = $${condor:condor_host}
[slap-parameter]
# Default values if not specified
condor_host = [$${slap-network-information:global-ipv6}]
collector_name = SLAPOS-CONDOR-POOL
role = manager
diskspace = 5
admin-email =
allowed-write = $${:condor_host}
#submit application
app-name = condor_test
description-file = ${description-file:location}/${description-file:filename}
executable = ${executable:location}/${executable:filename}
executable-name = ${executable:filename}
#user can specifie a list of input file, one per line
files =
condor-host = [$${slap-network-information:global-ipv6}]
condor-collector-name = SLAPOS-CONDOR-POOL
condor-role = manager
condor-diskspace = 5
condor-admin-email =
condor-allowed-write = $${:condor-host}
#submit application, using Json application list
condor-app-list = ${condor-application:list}
......@@ -30,7 +30,7 @@ md5sum = 9e9db6f4c5e38ce3fd45d43c2bf616a8
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-condor.cfg
output = ${buildout:directory}/template-condor.cfg
md5sum = 213b8c28be36f38103c7a100adb91d3f
md5sum = 42985a425369e4f93211cfaad21d8eb8
mode = 0644
[description-file]
......@@ -49,6 +49,9 @@ filename = simple
location = ${buildout:parts-directory}/${:_buildout_section_name_}
md5sum = c512f495cdd112bceb04feab7c909a10
[condor-application]
list = {"condor_test":{"description-file":"${description-file:location}/${description-file:filename}", "executable":"${executable:location}/${executable:filename}", "executable-name":"${executable:filename}", "files":{}}}
# Local development
[slapos.cookbook-repository]
recipe = plone.recipe.command
......@@ -64,6 +67,3 @@ recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[versions]
lxml = 2.3.6
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment