Commit 33744e40 authored by Sebastien Robin's avatar Sebastien Robin

add support for multi-repositories

- a list of repositories to synchronize can be provided
- the first repository is used for software
- transfer project revision with string like
  [repo x]=[commit count x]-[hash x],[repo y]=[commit count y][hash y]
- change related to new structure :
  recipe slapos.recipe.erp5testnode is inside slapos.cookbook egg
- allow authentification setting for several repositories
- drop support for svn
parent b15f406a
......@@ -83,36 +83,24 @@ class Recipe(BaseSlapRecipe):
git_binary=self.options['git_binary'],
software_root=CONFIG['software_root'],
working_directory=CONFIG['working_directory'],
vcs_repository=self.parameter_dict.get('vcs_repository'),
vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),),
node_quantity=self.parameter_dict.get('node_quantity', '1'),
branch=self.parameter_dict.get('branch', None),
test_suite_master_url=self.parameter_dict.get(
'test_suite_master_url', None),
test_suite_name=self.parameter_dict.get('test_suite_name'),
test_suite_title=self.parameter_dict.get('test_suite_title'),
project_title=self.parameter_dict.get('project_title'),
bin_directory=self.bin_directory,
foo='bar',
# botenvironemnt is splittable string of key=value to substitute
# environment of running bot
bot_environment=self.parameter_dict.get('bot_environment', ''),
partition_reference=CONFIG['partition_reference'],
environment=dict(PATH=os.environ['PATH']),
vcs_authentication_list=self.parameter_dict.get(
'vcs_authentication_list')
)
]))
def installLocalSvn(self):
svn_dict = dict(svn_binary = self.options['svn_binary'])
svn_dict.update(self.parameter_dict)
self._writeExecutable(os.path.join(self.bin_directory, 'svn'), """\
#!/bin/sh
%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
--non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
svnversion = os.path.join(self.bin_directory, 'svnversion')
if os.path.lexists(svnversion):
os.unlink(svnversion)
os.symlink(self.options['svnversion_binary'], svnversion)
def installLocalGit(self):
git_dict = dict(git_binary = self.options['git_binary'])
git_dict.update(self.parameter_dict)
......@@ -122,19 +110,18 @@ class Recipe(BaseSlapRecipe):
home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1])
print "home_directory : %r" % home_directory
git_dict.setdefault("git_server_name", "git.erp5.org")
if git_dict.get('vcs_username', None) is not None:
if git_dict.get('vcs_authentication_list', None) is not None:
netrc_file = open(os.path.join(home_directory, '.netrc'), 'w')
netrc_file.write("""
machine %(git_server_name)s
login %(vcs_username)s
password %(vcs_password)s""" % git_dict)
for vcs_authentication_dict in git_dict['vcs_authentication_list']:
netrc_file.write("""
machine %(host)s
login %(user_name)s
password %(password)s
""" % vcs_authentication_dict)
netrc_file.close()
def installLocalRepository(self):
if self.parameter_dict.get('vcs_repository').endswith('git'):
self.installLocalGit()
else:
self.installLocalSvn()
self.installLocalGit()
def installLocalZip(self):
zip = os.path.join(self.bin_directory, 'zip')
......
......@@ -60,39 +60,48 @@ slapos_controler = None
def run(args):
config = args[0]
slapgrid = None
branch = config.get('branch', None)
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid')
subprocess.check_call([config['git_binary'],
"config", "--global", "http.sslVerify", "false"])
previous_revision = None
run_software = True
# find what will be the path of the repository
repository_name = config['vcs_repository'].split('/')[-1].split('.')[0]
repository_path = os.path.join(config['working_directory'],repository_name)
config['repository_path'] = repository_path
sys.path.append(repository_path)
run_software = True
# Write our own software.cfg to use the local repository
custom_profile_path = os.path.join(config['working_directory'], 'software.cfg')
config['custom_profile_path'] = custom_profile_path
# create a profile in order to use the repository we already have
custom_profile = open(custom_profile_path, 'w')
profile_content = """
vcs_repository_list = config['vcs_repository_list']
profile_content = None
assert len(vcs_repository_list), "we must have at least one repository"
for vcs_repository in vcs_repository_list:
url = vcs_repository['url']
buildout_section_id = vcs_repository.get('buildout_section_id', None)
repository_id = buildout_section_id or \
url.split('/')[-1].split('.')[0]
repository_path = os.path.join(config['working_directory'],repository_id)
vcs_repository['repository_id'] = repository_id
vcs_repository['repository_path'] = repository_path
if profile_content is None:
profile_content = """
[buildout]
extends = %(software_config_path)s
[%(repository_name)s]
""" % {'software_config_path': os.path.join(repository_path,
config['profile_path'])}
if not(buildout_section_id is None):
profile_content += """
[%(buildout_section_id)s]
repository = %(repository_path)s
""" % {'software_config_path': os.path.join(repository_path,
config['profile_path']),
'repository_name': repository_name,
'repository_path' : repository_path}
if branch is not None:
profile_content += "\nbranch = %s" % branch
branch = %(branch)s
""" % {'buildout_section_id': buildout_section_id,
'repository_path' : repository_path,
'branch' : vcs_repository.get('branch','')}
custom_profile = open(custom_profile_path, 'w')
custom_profile.write(profile_content)
custom_profile.close()
config['repository_path'] = repository_path
sys.path.append(repository_path)
retry_software = False
try:
while True:
......@@ -104,19 +113,24 @@ repository = %(repository_path)s
except:
pass
process_group_pid_set.clear()
full_revision_list = []
# Make sure we have local repository
if not os.path.exists(repository_path):
parameter_list = [config['git_binary'], 'clone',
config['vcs_repository']]
if branch is not None:
parameter_list.extend(['-b',branch])
parameter_list.append(repository_path)
subprocess.check_call(parameter_list)
# XXX this looks like to not wait the end of the command
# Make sure we have local repository
updater = Updater(repository_path, git_binary=config['git_binary'])
updater.checkout()
revision = updater.getRevision()
for vcs_repository in vcs_repository_list:
repository_path = vcs_repository['repository_path']
repository_id = vcs_repository['repository_id']
if not os.path.exists(repository_path):
parameter_list = [config['git_binary'], 'clone',
vcs_repository['url']]
if vcs_repository.get('branch') is not None:
parameter_list.extend(['-b',vcs_repository.get('branch')])
parameter_list.append(repository_path)
subprocess.check_call(parameter_list)
# Make sure we have local repository
updater = Updater(repository_path, git_binary=config['git_binary'])
updater.checkout()
revision = "-".join(updater.getRevision())
full_revision_list.append('%s=%s' % (repository_id, revision))
revision = ','.join(full_revision_list)
if previous_revision == revision:
time.sleep(120)
if not(retry_software):
......@@ -138,16 +152,19 @@ repository = %(repository_path)s
assert master.getProtocolRevision() == 1
test_result = safeRpcCall(master.createTestResult,
config['test_suite_name'], revision, [],
False, config['test_suite_title'])
False, config['test_suite_title'], config['project_title'])
print "testnode, test_result : %r" % (test_result,)
if test_result:
test_result_path, test_revision = test_result
if revision != test_revision:
# other testnodes on other boxes are already ready to test another
# revision
updater = Updater(repository_path, git_binary=config['git_binary'],
revision=test_revision)
updater.checkout()
for i, repository_revision in enumerate(test_revision.split(',')):
vcs_repository = vcs_repository_list[i]
repository_path = vcs_repository['repository_path']
# other testnodes on other boxes are already ready to test another
# revision
updater = Updater(repository_path, git_binary=config['git_binary'],
revision=repository_revision.split('-')[1])
updater.checkout()
# Now prepare the installation of SlapOS
slapos_controler = SlapOSControler(config,
......@@ -169,24 +186,21 @@ repository = %(repository_path)s
# update repositories downloaded by buildout. Later we should get
# from master a list of repositories
repository_path_list = glob(os.path.join(config['software_root'],
'*', 'parts', 'git_repository', '*'))
assert len(repository_path_list) >= 0
for repository_path in repository_path_list:
updater = Updater(repository_path, git_binary=config['git_binary'])
updater.checkout()
if os.path.split(repository_path)[-1] == repository_name:
# redo checkout with good revision, the previous one is used
# to pull last code
updater = Updater(repository_path, git_binary=config['git_binary'],
revision=revision)
updater.checkout()
# calling dist/externals is only there for backward compatibility,
# the code will be removed soon
if os.path.exists(os.path.join(repository_path, 'dist/externals.py')):
process = subprocess.Popen(['dist/externals.py'],
cwd=repository_path)
process.wait()
# The section below seems useless since we override buildout
# configuration to use local checkouts
#repository_path_list = glob(os.path.join(config['software_root'],
#'*', 'parts', 'git_repository', '*'))
#assert len(repository_path_list) >= 0
#for repository_path in repository_path_list:
#updater = Updater(repository_path, git_binary=config['git_binary'])
#updater.checkout()
#if os.path.split(repository_path)[-1] == repository_name:
## redo checkout with good revision, the previous one is used
## to pull last code
#updater = Updater(repository_path, git_binary=config['git_binary'],
#revision=revision)
#updater.checkout()
partition_path = os.path.join(config['instance_root'],
config['partition_reference'])
......
......@@ -6,7 +6,7 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[testnode]
recipe = erp5.recipe.testnode
recipe = slapos.cookbook:erp5testnode
buildbot_binary = ${buildout:bin-directory}/buildbot
slapgrid_partition_binary = ${buildout:bin-directory}/slapgrid-cp
......
......@@ -46,12 +46,11 @@ zc.buildout = 1.5.3-dev-SlapOS-001
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
erp5.recipe.testnode
slapos.core
slapos.cookbook
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 0b44b72c9e21d760f7e27a89e9d10187
output = ${buildout:directory}/template.cfg
mode = 0644
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment