Commit 092c7c42 authored by Alain Takoudjou's avatar Alain Takoudjou

Use Json parameters in condor recipe

parent 1601b8fa
...@@ -32,6 +32,7 @@ import filecmp ...@@ -32,6 +32,7 @@ import filecmp
import urlparse import urlparse
import shutil import shutil
import re import re
import json
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
"""Deploy a fully operational condor architecture.""" """Deploy a fully operational condor architecture."""
...@@ -219,67 +220,69 @@ class AppSubmit(GenericBaseRecipe): ...@@ -219,67 +220,69 @@ class AppSubmit(GenericBaseRecipe):
def getFiles(self): def getFiles(self):
"""This is used to download app files if necessary and update options values""" """This is used to download app files if necessary and update options values"""
self.options['file-number'] = 0 app_list = json.loads(self.options['condor-app-list'])
if self.options['files']: if not app_list:
files_list = self.options['files'].splitlines() return None
files_list = [f for f in files_list if f] #remove empty elements for app in app_list:
self.options['file-number'] = len(files_list) if app_list[app].get('files', None):
for i in range(self.options['file-number']): file_list = app_list[app]['files']
value = files_list[i].strip() for file in file_list:
pos = str(i) if file and (file.startswith('http') or file.startswith('ftp')):
if value and (value.startswith('http') or value.startswith('ftp')): file_list[file] = self.download(file_list[file])
self.options['name_'+pos] = os.path.basename(urlparse.urlparse(value)[2]) os.chmod(file_list[file], 0600)
self.options['file_'+pos] = self.download(value) else:
os.chmod(self.options['file_'+pos], 0600) app_list[app]['files'] = {}
else:
self.options['file_'+pos] = value executable = app_list[app].get('executable', '')
executable = self.options['executable'] if executable and (executable.startswith('http') or executable.startswith('ftp')):
if executable and (executable.startswith('http') or executable.startswith('ftp')): app_list[app]['executable'] = self.download(executable,
self.options['executable'] = self.download(executable, app_list[app]['executable-name'])
self.options['executable-name'].strip()) os.chmod(app_list[app]['executable-name'], 0700)
os.chmod(self.options['executable'], 0700) submit_file = app_list[app].get('description-file', '')
submit_file = self.options['description-file'] if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')): app_list[app]['description-file'] = self.download(submit_file, 'submit')
self.options['description-file'] = self.download(submit_file, 'submit') os.chmod(app_list[app]['description-file'], 0600)
os.chmod(self.options['description-file'], 0600)
return app_list
def install(self): def install(self):
path_list = [] path_list = []
#check if curent condor instance is an condor master #check if curent condor instance is an condor master
if self.options['machine-role'].strip() != "manager": if self.options['machine-role'].strip() != "manager":
print "ERROR: cannot submit a job to Condor worker instance" raise Exception("Cannot submit a job to Condor worker instance")
return []
#Setup directory #Setup directory
jobdir = self.options['job-dir'].strip() jobdir = self.options['job-dir'].strip()
appdir = os.path.join(jobdir, self.options['app-name'].strip())
submitfile = os.path.join(appdir, 'submit')
appname = self.options['app-name'].strip()
if not os.path.exists(jobdir): if not os.path.exists(jobdir):
os.mkdir(jobdir) os.mkdir(jobdir)
if not os.path.exists(appdir): app_list = self.getFiles()
os.mkdir(appdir) for appname in app_list:
self.getFiles() appdir = os.path.join(jobdir, appname)
self.copy_file(self.options['executable'], if not os.path.exists(appdir):
os.path.join(appdir, self.options['executable-name'].strip()) os.mkdir(appdir)
) submitfile = os.path.join(appdir, 'submit')
install = self.copy_file(self.options['description-file'], submitfile)
sig_install = os.path.join(appdir, '.install') self.copy_file(app_list[appname]['executable'],
if install: os.path.join(appdir, app_list[appname]['executable-name'])
with open(sig_install, 'w') as f: )
f.write('to_install') install = self.copy_file(app_list[appname]['description-file'], submitfile)
for i in range(self.options['file-number']): sig_install = os.path.join(appdir, '.install')
destination = os.path.join(appdir, self.options['name_'+str(i)]) if install:
if os.path.exists(destination): with open(sig_install, 'w') as f:
os.unlink(destination) f.write('to_install')
os.symlink(self.options['file_'+str(i)], destination) for file in app_list[appname]['files']:
#generate wrapper for submitting job destination = os.path.join(appdir, file)
condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit') if os.path.exists(destination):
parameter = dict(submit=condor_submit, sig_install=sig_install, os.unlink(destination)
submit_file='submit', os.symlink(app_list[appname]['files'][file], destination)
appname=appname, appdir=appdir) #generate wrapper for submitting job
submit_job = self.createPythonScript( condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
os.path.join(self.options['wrapper-dir'].strip(), appname), parameter = dict(submit=condor_submit, sig_install=sig_install,
'%s.configure.submitJob' % __name__, parameter submit_file='submit',
) appname=appname, appdir=appdir)
path_list.append(submit_job) submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
'%s.configure.submitJob' % __name__, parameter
)
path_list.append(submit_job)
return path_list return path_list
\ No newline at end of file
...@@ -24,7 +24,7 @@ srv = $${buildout:directory}/srv/ ...@@ -24,7 +24,7 @@ srv = $${buildout:directory}/srv/
[basedirectory] [basedirectory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/ log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/ scripts = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/ run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/ backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/ promises = $${rootdirectory:etc}/promise/
...@@ -35,7 +35,7 @@ wrapper = $${basedirectory:run}/condor ...@@ -35,7 +35,7 @@ wrapper = $${basedirectory:run}/condor
[condor] [condor]
recipe = slapos.cookbook:condor recipe = slapos.cookbook:condor
wrapper-dir = $${basedirectory:services} wrapper-dir = $${basedirectory:scripts}
ip = $${slap-network-information:global-ipv6} ip = $${slap-network-information:global-ipv6}
package = ${condor:location} package = ${condor:location}
rootdirectory = $${buildout:directory} rootdirectory = $${buildout:directory}
...@@ -50,46 +50,36 @@ environment = ...@@ -50,46 +50,36 @@ environment =
LD_LIBRARY_PATH=${libexpat:location}/lib:${kerberos:location}/lib:${openldap:location}/lib:${zlib:location}/lib LD_LIBRARY_PATH=${libexpat:location}/lib:${kerberos:location}/lib:${openldap:location}/lib:${zlib:location}/lib
PATH=${perl:location}/bin:${java:location}/bin:${kerberos:location}/bin:${openldap:location}/bin:%(PATH)s PATH=${perl:location}/bin:${java:location}/bin:${kerberos:location}/bin:${openldap:location}/bin:%(PATH)s
HOME=$${rootdirectory:condor} HOME=$${rootdirectory:condor}
HOSTNAME=$${slap-parameter:condor_host} HOSTNAME=$${slap-parameter:condor-host}
#Condor user parameter #Condor user parameter
condor_host = $${slap-parameter:condor_host} condor_host = $${slap-parameter:condor-host}
collector_name = $${slap-parameter:collector_name} collector_name = $${slap-parameter:condor-collector-name}
#Condor machine role: worker=submit,execute manager=manager,submit #Condor machine role: worker=submit,execute manager=manager,submit
machine-role = $${slap-parameter:role} machine-role = $${slap-parameter:condor-role}
disk-space = $${slap-parameter:diskspace} disk-space = $${slap-parameter:condor-diskspace}
allowed-write = $${slap-parameter:allowed-write} allowed-write = $${slap-parameter:condor-allowed-write}
admin-email = $${slap-parameter:admin-email} admin-email = $${slap-parameter:condor-admin-email}
[app-submit] [app-submit]
<= condor <= condor
recipe = slapos.cookbook:condor.submit recipe = slapos.cookbook:condor.submit
app-name = $${slap-parameter:app-name} condor-app-list = $${slap-parameter:condor-app-list}
#user can specifie a list files to use in application, one per line
files = $${slap-parameter:files}
description-file = $${slap-parameter:description-file}
executable = $${slap-parameter:executable}
executable-name = $${slap-parameter:executable-name}
# Publish all instance parameters (url of instance) # Publish all instance parameters (url of instance)
[publish-connection-informations] [publish-connection-informations]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
ipv6_address = $${condor:ip} condor_ipv6_address = $${condor:ip}
instance_type = $${condor:machine-role} condor_instance_type = $${condor:machine-role}
condor_host = $${condor:condor_host} condor_host = $${condor:condor_host}
[slap-parameter] [slap-parameter]
# Default values if not specified # Default values if not specified
condor_host = [$${slap-network-information:global-ipv6}] condor-host = [$${slap-network-information:global-ipv6}]
collector_name = SLAPOS-CONDOR-POOL condor-collector-name = SLAPOS-CONDOR-POOL
role = manager condor-role = manager
diskspace = 5 condor-diskspace = 5
admin-email = condor-admin-email =
allowed-write = $${:condor_host} condor-allowed-write = $${:condor-host}
#submit application #submit application, using Json application list
app-name = condor_test condor-app-list = ${condor-application:list}
description-file = ${description-file:location}/${description-file:filename}
executable = ${executable:location}/${executable:filename}
executable-name = ${executable:filename}
#user can specifie a list of input file, one per line
files =
...@@ -30,7 +30,7 @@ md5sum = 9e9db6f4c5e38ce3fd45d43c2bf616a8 ...@@ -30,7 +30,7 @@ md5sum = 9e9db6f4c5e38ce3fd45d43c2bf616a8
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-condor.cfg url = ${:_profile_base_location_}/instance-condor.cfg
output = ${buildout:directory}/template-condor.cfg output = ${buildout:directory}/template-condor.cfg
md5sum = 213b8c28be36f38103c7a100adb91d3f md5sum = 42985a425369e4f93211cfaad21d8eb8
mode = 0644 mode = 0644
[description-file] [description-file]
...@@ -49,6 +49,9 @@ filename = simple ...@@ -49,6 +49,9 @@ filename = simple
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
md5sum = c512f495cdd112bceb04feab7c909a10 md5sum = c512f495cdd112bceb04feab7c909a10
[condor-application]
list = {"condor_test":{"description-file":"${description-file:location}/${description-file:filename}", "executable":"${executable:location}/${executable:filename}", "executable-name":"${executable:filename}", "files":{}}}
# Local development # Local development
[slapos.cookbook-repository] [slapos.cookbook-repository]
recipe = plone.recipe.command recipe = plone.recipe.command
...@@ -64,6 +67,3 @@ recipe = plone.recipe.command ...@@ -64,6 +67,3 @@ recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[versions]
lxml = 2.3.6
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment