Skip to content

slapos
slapos
Commit 3a4e5da9 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

condor: move component to unstable, remove recipe

parent b3082041
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 2963 deletions
setup.py
View file @ 3a4e5da9
......@@ -90,8 +90,6 @@ setup(name=name,
'check_parameter = slapos.recipe.check_parameter:Recipe',
'cloud9 = slapos.recipe.cloud9:Recipe',
'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe',
'condor = slapos.recipe.condor:Recipe',
'condor.submit = slapos.recipe.condor:AppSubmit',
'copyfilelist = slapos.recipe.copyfilelist:Recipe',
'cron = slapos.recipe.dcron:Recipe',
'cron.d = slapos.recipe.dcron:Part',
......
slapos/recipe/condor/__init__.py deleted 100644 → 0
View file @ b3082041
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import zc.buildout
import filecmp
import shutil
import re
import json
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational condor architecture."""
def __init__(self, buildout, name, options):
self.environ = {}
self.role = ''
environment_section = options.get('environment-section', '').strip()
if environment_section and environment_section in buildout:
# Use environment variables from the designated config section.
self.environ.update(buildout[environment_section])
for variable in options.get('environment', '').splitlines():
if variable.strip():
try:
key, value = variable.split('=', 1)
self.environ[key.strip()] = value
except ValueError:
raise zc.buildout.UserError('Invalid environment variable definition: %s', variable)
# Extrapolate the environment variables using values from the current
# environment.
for key in self.environ:
self.environ[key] = self.environ[key] % os.environ
return GenericBaseRecipe.__init__(self, buildout, name, options)
def _options(self, options):
#Path of condor compiled package
self.package = options['package'].strip()
self.rootdir = options['rootdirectory'].strip()
#Other condor dependances
self.javabin = options['java-bin'].strip()
self.dash = options['dash'].strip()
#Directory to deploy condor
self.prefix = options['rootdirectory'].strip()
self.localdir = options['local-dir'].strip()
self.wrapperdir = options['wrapper-dir'].strip()
self.wrapper_bin = options['bin'].strip()
self.wrapper_sbin = options['sbin'].strip()
self.diskspace = options['disk-space'].strip()
self.ipv6 = options['ip'].strip()
self.condor_host = options['condor_host'].strip()
self.collector_name = options['collector_name'].strip()
self.host_list = self.options.get('allowed-write', '*')
self.email = self.options.get('admin-email', "root@$(FULL_HOSTNAME)")
def install(self):
path_list = []
#get UID and GID for current slapuser
stat_info = os.stat(self.rootdir)
slapuser = str(stat_info.st_uid)+"."+str(stat_info.st_gid)
domain_name = 'slapos%s.com' % stat_info.st_uid
#Configure condor
configure_script = os.path.join(self.package, 'condor_configure')
install_args = [configure_script, '--install='+self.package,
'--prefix='+self.prefix, '--overwrite', '--verbose',
'--local-dir='+self.localdir] #--ignore-missing-libs
if self.options['machine-role'].strip() == "manager":
self.role = "manager,submit"
elif self.options['machine-role'].strip() == "worker":
self.role = "execute"
install_args += ['--central-manager='+self.condor_host]
install_args += ['--type='+self.role]
configure = subprocess.Popen(install_args, env=self.environ,
stdout=subprocess.PIPE)
configure.communicate()[0]
if configure.returncode is None or configure.returncode != 0:
return path_list
#Generate condor_configure file
condor_config = os.path.join(self.rootdir, 'etc/condor_config')
config_local = os.path.join(self.localdir, 'condor_config.local')
condor_configure = dict(condor_host=self.condor_host, releasedir=self.prefix,
localdir=self.localdir, config_local=config_local,
slapuser=slapuser, ipv6=self.ipv6,
diskspace=self.diskspace, javabin=self.javabin,
host_list=self.host_list, collector_name=self.collector_name,
email=self.email, domain_name=domain_name)
destination = os.path.join(condor_config)
config = self.createFile(destination,
self.substituteTemplate(self.getTemplateFilename('condor_config.generic'),
condor_configure))
path_list.append(config)
#Search if is needed to update condor_config.local file
find = re.search('NETWORK_INTERFACE[\s]*=[\s]*(%s)' % self.ipv6,
open(config_local, 'r').read())
if not find:
#update condor_config.local
with open(config_local, 'a') as f:
if self.role == "execute":
f.write("\nSTART = TRUE")
f.write("\nCOLLECTOR_NAME = %s\n \nNETWORK_INTERFACE=%s" %
(self.collector_name, self.ipv6))
#create condor binary launcher for slapos
if not os.path.exists(self.wrapper_bin):
os.makedirs(self.wrapper_bin, int('0o744', 8))
if not os.path.exists(self.wrapper_sbin):
os.makedirs(self.wrapper_sbin, int('0o744', 8))
#generate script for each file in prefix/bin
for binary in os.listdir(self.prefix+'/bin'):
wrapper_location = os.path.join(self.wrapper_bin, binary)
current_exe = os.path.join(self.prefix, 'bin', binary)
wrapper = open(wrapper_location, 'w')
content = """#!%s
export LD_LIBRARY_PATH=%s
export PATH=%s
export CONDOR_CONFIG=%s
export CONDOR_LOCATION=%s
export CONDOR_IDS=%s
export HOME=%s
export HOSTNAME=%s
exec %s $*""" % (self.dash,
self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
condor_config, self.prefix, slapuser, self.environ['HOME'],
self.environ['HOSTNAME'], current_exe)
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0o744)
#generate script for each file in prefix/sbin
for binary in os.listdir(self.prefix+'/sbin'):
wrapper_location = os.path.join(self.wrapper_sbin, binary)
current_exe = os.path.join(self.prefix, 'sbin', binary)
wrapper = open(wrapper_location, 'w')
content = """#!%s
export LD_LIBRARY_PATH=%s
export PATH=%s
export CONDOR_CONFIG=%s
export CONDOR_LOCATION=%s
export CONDOR_IDS=%s
export HOME=%s
export HOSTNAME=%s
exec %s $*""" % (self.dash,
self.environ['LD_LIBRARY_PATH'], self.environ['PATH'],
condor_config, self.prefix, slapuser, self.environ['HOME'],
self.environ['HOSTNAME'], current_exe)
wrapper.write(content)
wrapper.close()
path_list.append(wrapper_location)
os.chmod(wrapper_location, 0o744)
#generate script for start condor
wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_condor'),
__name__ + '.configure.condorStart',
(os.path.join(self.wrapper_sbin, 'condor_reconfig'),
os.path.join(self.wrapper_sbin, 'condor_master'))
)
path_list.append(wrapper)
return path_list
class AppSubmit(GenericBaseRecipe):
"""Submit a condor job into an existing Condor master instance"""
def download(self, url, filename=None, md5sum=None):
cache = os.path.join(self.options['rootdirectory'].strip(), 'tmp')
if not os.path.exists(cache):
os.mkdir(cache)
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(url, md5sum)
if filename:
name = os.path.join(cache, filename)
os.rename(path, name)
return name
return path
def copy_file(self, source, dest):
""""Copy file with source to dest with auto replace
return True if file has been copied and dest ha been replaced
"""
result = False
if source and os.path.exists(source):
if os.path.exists(dest):
if filecmp.cmp(dest, source):
return False
os.unlink(dest)
result = True
shutil.copy(source, dest)
return result
def getFiles(self):
"""This is used to download app files if necessary and update options values"""
app_list = json.loads(self.options['condor-app-list'])
if not app_list:
return None
for app in app_list:
if app_list[app].get('files', None):
file_list = app_list[app]['files']
for file in file_list:
if file and (file.startswith('http') or file.startswith('ftp')):
file_list[file] = self.download(file_list[file])
os.chmod(file_list[file], 0o600)
else:
app_list[app]['files'] = {}
executable = app_list[app].get('executable', '')
if executable and (executable.startswith('http') or executable.startswith('ftp')):
app_list[app]['executable'] = self.download(executable,
app_list[app]['executable-name'])
os.chmod(app_list[app]['executable-name'], 0o700)
submit_file = app_list[app].get('description-file', '')
if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
app_list[app]['description-file'] = self.download(submit_file, 'submit')
os.chmod(app_list[app]['description-file'], 0o600)
return app_list
def install(self):
path_list = []
#check if curent condor instance is an condor master
if self.options['machine-role'].strip() != "manager":
raise Exception("Cannot submit a job to Condor worker instance")
#Setup directory
jobdir = self.options['job-dir'].strip()
if not os.path.exists(jobdir):
os.mkdir(jobdir)
app_list = self.getFiles()
for appname in app_list:
appdir = os.path.join(jobdir, appname)
if not os.path.exists(appdir):
os.mkdir(appdir)
submitfile = os.path.join(appdir, 'submit')
self.copy_file(app_list[appname]['executable'],
os.path.join(appdir, app_list[appname]['executable-name'])
)
install = self.copy_file(app_list[appname]['description-file'], submitfile)
sig_install = os.path.join(appdir, '.install')
if install:
with open(sig_install, 'w') as f:
f.write('to_install')
for file in app_list[appname]['files']:
destination = os.path.join(appdir, file)
if os.path.exists(destination):
os.unlink(destination)
os.symlink(app_list[appname]['files'][file], destination)
#generate wrapper for submitting job
submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
__name__ + '.configure.submitJob',
(os.path.join(self.options['bin'].strip(), 'condor_submit'),
'submit', appdir, appname, sig_install)
)
path_list.append(submit_job)
return path_list
\ No newline at end of file
slapos/recipe/condor/configure.py deleted 100644 → 0
View file @ b3082041
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import subprocess
import time
def submitJob(submit, submit_file, appdir, appname, sig_install):
"""Run condor_submit (if needed) for job deployment"""
time.sleep(10)
print "Check if needed to submit %s job's" % appname
if not os.path.exists(sig_install):
print "Nothing for install or update...Exited"
return
# '-a', "log = out.log", '-a', "error = error.log",
launch_args = submit, '-verbose', submit_file
process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=appdir)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute condor_submit.\nThe error was: %s" % result
else:
os.unlink(sig_install)
def condorStart(condor_reconfig, start_bin):
"""Start Condor if deamons is currently stopped"""
if subprocess.call(condor_reconfig):
#process failled to reconfig condor that mean that condor deamons is not curently started
subprocess.call(start_bin)
slapos/recipe/condor/template/condor_config.generic deleted 100644 → 0
View file @ b3082041
This source diff could not be displayed because it is too large. You can view the blob instead.
slapos/recipe/condor/template/condor_config.local deleted 100644 → 0
View file @ b3082041
## Pathnames:
## Where have you installed the bin, sbin and lib condor directories?
RELEASE_DIR = %(releasedir)s
## Where is the local condor directory for each host?
## This is where the local config file(s), logs and
## spool/execute directories are located
LOCAL_DIR = %(localdir)s
## Mail parameters:
## When something goes wrong with condor at your site, who should get
## the email?
CONDOR_ADMIN =
## The user/group ID <uid>.<gid> of the "Condor" user.
## (this can also be specified in the environment)
## Note: the CONDOR_IDS setting is ignored on Win32 platforms
CONDOR_IDS = %(slapuser)s
## Condor needs to create a few lock files to synchronize access to
## various log files. Because of problems we've had with network
## filesystems and file locking over the years, we HIGHLY recommend
## that you put these lock files on a local partition on each
## machine. If you don't have your LOCAL_DIR on a local partition,
## be sure to change this entry. Whatever user (or group) condor is
## running as needs to have write access to this directory. If
## you're not running as root, this is whatever user you started up
## the condor_master as. If you are running as root, and there's a
## condor account, it's probably condor. Otherwise, it's whatever
## you've set in the CONDOR_IDS environment variable. See the Admin
## manual for details on this.
LOCK = /tmp/condor-lock.$(HOSTNAME)0.829500835462571
DAEMON_LIST = MASTER, SCHEDD, STARTD
## Java parameters:
## If you would like this machine to be able to run Java jobs,
## then set JAVA to the path of your JVM binary. If you are not
## interested in Java, there is no harm in leaving this entry
## empty or incorrect.
JAVA = %(java-bin)s
JAVA_MAXHEAP_ARGUMENT = -Xmx1024m
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7