Commit 9af98961 authored by Sebastien Robin's avatar Sebastien Robin Committed by Julien Muchembled

add support for multi-repositories

- a list of repositories to synchronize can be provided
- the first repository is used for software
- transfer project revision with string like
  [repo x]=[commit count x]-[hash x],[repo y]=[commit count y][hash y]
- change related to new structure :
  recipe slapos.recipe.erp5testnode is inside slapos.cookbook egg
- allow authentification setting for several repositories
- drop support for svn
parent 3005c48d
...@@ -83,36 +83,24 @@ class Recipe(BaseSlapRecipe): ...@@ -83,36 +83,24 @@ class Recipe(BaseSlapRecipe):
git_binary=self.options['git_binary'], git_binary=self.options['git_binary'],
software_root=CONFIG['software_root'], software_root=CONFIG['software_root'],
working_directory=CONFIG['working_directory'], working_directory=CONFIG['working_directory'],
vcs_repository=self.parameter_dict.get('vcs_repository'), vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),),
node_quantity=self.parameter_dict.get('node_quantity', '1'), node_quantity=self.parameter_dict.get('node_quantity', '1'),
branch=self.parameter_dict.get('branch', None),
test_suite_master_url=self.parameter_dict.get( test_suite_master_url=self.parameter_dict.get(
'test_suite_master_url', None), 'test_suite_master_url', None),
test_suite_name=self.parameter_dict.get('test_suite_name'), test_suite_name=self.parameter_dict.get('test_suite_name'),
test_suite_title=self.parameter_dict.get('test_suite_title'), test_suite_title=self.parameter_dict.get('test_suite_title'),
project_title=self.parameter_dict.get('project_title'),
bin_directory=self.bin_directory, bin_directory=self.bin_directory,
foo='bar',
# botenvironemnt is splittable string of key=value to substitute # botenvironemnt is splittable string of key=value to substitute
# environment of running bot # environment of running bot
bot_environment=self.parameter_dict.get('bot_environment', ''), bot_environment=self.parameter_dict.get('bot_environment', ''),
partition_reference=CONFIG['partition_reference'], partition_reference=CONFIG['partition_reference'],
environment=dict(PATH=os.environ['PATH']), environment=dict(PATH=os.environ['PATH']),
vcs_authentication_list=self.parameter_dict.get(
'vcs_authentication_list')
) )
])) ]))
def installLocalSvn(self):
svn_dict = dict(svn_binary = self.options['svn_binary'])
svn_dict.update(self.parameter_dict)
self._writeExecutable(os.path.join(self.bin_directory, 'svn'), """\
#!/bin/sh
%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
--non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
svnversion = os.path.join(self.bin_directory, 'svnversion')
if os.path.lexists(svnversion):
os.unlink(svnversion)
os.symlink(self.options['svnversion_binary'], svnversion)
def installLocalGit(self): def installLocalGit(self):
git_dict = dict(git_binary = self.options['git_binary']) git_dict = dict(git_binary = self.options['git_binary'])
git_dict.update(self.parameter_dict) git_dict.update(self.parameter_dict)
...@@ -122,19 +110,18 @@ class Recipe(BaseSlapRecipe): ...@@ -122,19 +110,18 @@ class Recipe(BaseSlapRecipe):
home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1]) home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1])
print "home_directory : %r" % home_directory print "home_directory : %r" % home_directory
git_dict.setdefault("git_server_name", "git.erp5.org") git_dict.setdefault("git_server_name", "git.erp5.org")
if git_dict.get('vcs_username', None) is not None: if git_dict.get('vcs_authentication_list', None) is not None:
netrc_file = open(os.path.join(home_directory, '.netrc'), 'w') netrc_file = open(os.path.join(home_directory, '.netrc'), 'w')
netrc_file.write(""" for vcs_authentication_dict in git_dict['vcs_authentication_list']:
machine %(git_server_name)s netrc_file.write("""
login %(vcs_username)s machine %(host)s
password %(vcs_password)s""" % git_dict) login %(user_name)s
password %(password)s
""" % vcs_authentication_dict)
netrc_file.close() netrc_file.close()
def installLocalRepository(self): def installLocalRepository(self):
if self.parameter_dict.get('vcs_repository').endswith('git'): self.installLocalGit()
self.installLocalGit()
else:
self.installLocalSvn()
def installLocalZip(self): def installLocalZip(self):
zip = os.path.join(self.bin_directory, 'zip') zip = os.path.join(self.bin_directory, 'zip')
......
...@@ -60,39 +60,48 @@ slapos_controler = None ...@@ -60,39 +60,48 @@ slapos_controler = None
def run(args): def run(args):
config = args[0] config = args[0]
slapgrid = None slapgrid = None
branch = config.get('branch', None)
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run', supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid') 'supervisord.pid')
subprocess.check_call([config['git_binary'], subprocess.check_call([config['git_binary'],
"config", "--global", "http.sslVerify", "false"]) "config", "--global", "http.sslVerify", "false"])
previous_revision = None previous_revision = None
run_software = True
# find what will be the path of the repository
repository_name = config['vcs_repository'].split('/')[-1].split('.')[0]
repository_path = os.path.join(config['working_directory'],repository_name)
config['repository_path'] = repository_path
sys.path.append(repository_path)
run_software = True
# Write our own software.cfg to use the local repository # Write our own software.cfg to use the local repository
custom_profile_path = os.path.join(config['working_directory'], 'software.cfg') custom_profile_path = os.path.join(config['working_directory'], 'software.cfg')
config['custom_profile_path'] = custom_profile_path config['custom_profile_path'] = custom_profile_path
vcs_repository_list = config['vcs_repository_list']
# create a profile in order to use the repository we already have profile_content = None
custom_profile = open(custom_profile_path, 'w') assert len(vcs_repository_list), "we must have at least one repository"
profile_content = """ for vcs_repository in vcs_repository_list:
url = vcs_repository['url']
buildout_section_id = vcs_repository.get('buildout_section_id', None)
repository_id = buildout_section_id or \
url.split('/')[-1].split('.')[0]
repository_path = os.path.join(config['working_directory'],repository_id)
vcs_repository['repository_id'] = repository_id
vcs_repository['repository_path'] = repository_path
if profile_content is None:
profile_content = """
[buildout] [buildout]
extends = %(software_config_path)s extends = %(software_config_path)s
""" % {'software_config_path': os.path.join(repository_path,
[%(repository_name)s] config['profile_path'])}
if not(buildout_section_id is None):
profile_content += """
[%(buildout_section_id)s]
repository = %(repository_path)s repository = %(repository_path)s
""" % {'software_config_path': os.path.join(repository_path, branch = %(branch)s
config['profile_path']), """ % {'buildout_section_id': buildout_section_id,
'repository_name': repository_name, 'repository_path' : repository_path,
'repository_path' : repository_path} 'branch' : vcs_repository.get('branch','')}
if branch is not None:
profile_content += "\nbranch = %s" % branch custom_profile = open(custom_profile_path, 'w')
custom_profile.write(profile_content) custom_profile.write(profile_content)
custom_profile.close() custom_profile.close()
config['repository_path'] = repository_path
sys.path.append(repository_path)
retry_software = False retry_software = False
try: try:
while True: while True:
...@@ -104,19 +113,24 @@ repository = %(repository_path)s ...@@ -104,19 +113,24 @@ repository = %(repository_path)s
except: except:
pass pass
process_group_pid_set.clear() process_group_pid_set.clear()
full_revision_list = []
# Make sure we have local repository # Make sure we have local repository
if not os.path.exists(repository_path): for vcs_repository in vcs_repository_list:
parameter_list = [config['git_binary'], 'clone', repository_path = vcs_repository['repository_path']
config['vcs_repository']] repository_id = vcs_repository['repository_id']
if branch is not None: if not os.path.exists(repository_path):
parameter_list.extend(['-b',branch]) parameter_list = [config['git_binary'], 'clone',
parameter_list.append(repository_path) vcs_repository['url']]
subprocess.check_call(parameter_list) if vcs_repository.get('branch') is not None:
# XXX this looks like to not wait the end of the command parameter_list.extend(['-b',vcs_repository.get('branch')])
# Make sure we have local repository parameter_list.append(repository_path)
updater = Updater(repository_path, git_binary=config['git_binary']) subprocess.check_call(parameter_list)
updater.checkout() # Make sure we have local repository
revision = updater.getRevision() updater = Updater(repository_path, git_binary=config['git_binary'])
updater.checkout()
revision = "-".join(updater.getRevision())
full_revision_list.append('%s=%s' % (repository_id, revision))
revision = ','.join(full_revision_list)
if previous_revision == revision: if previous_revision == revision:
time.sleep(120) time.sleep(120)
if not(retry_software): if not(retry_software):
...@@ -138,16 +152,19 @@ repository = %(repository_path)s ...@@ -138,16 +152,19 @@ repository = %(repository_path)s
assert master.getProtocolRevision() == 1 assert master.getProtocolRevision() == 1
test_result = safeRpcCall(master.createTestResult, test_result = safeRpcCall(master.createTestResult,
config['test_suite_name'], revision, [], config['test_suite_name'], revision, [],
False, config['test_suite_title']) False, config['test_suite_title'], config['project_title'])
print "testnode, test_result : %r" % (test_result,) print "testnode, test_result : %r" % (test_result,)
if test_result: if test_result:
test_result_path, test_revision = test_result test_result_path, test_revision = test_result
if revision != test_revision: if revision != test_revision:
# other testnodes on other boxes are already ready to test another for i, repository_revision in enumerate(test_revision.split(',')):
# revision vcs_repository = vcs_repository_list[i]
updater = Updater(repository_path, git_binary=config['git_binary'], repository_path = vcs_repository['repository_path']
revision=test_revision) # other testnodes on other boxes are already ready to test another
updater.checkout() # revision
updater = Updater(repository_path, git_binary=config['git_binary'],
revision=repository_revision.split('-')[1])
updater.checkout()
# Now prepare the installation of SlapOS # Now prepare the installation of SlapOS
slapos_controler = SlapOSControler(config, slapos_controler = SlapOSControler(config,
...@@ -169,24 +186,21 @@ repository = %(repository_path)s ...@@ -169,24 +186,21 @@ repository = %(repository_path)s
# update repositories downloaded by buildout. Later we should get # update repositories downloaded by buildout. Later we should get
# from master a list of repositories # from master a list of repositories
repository_path_list = glob(os.path.join(config['software_root'],
'*', 'parts', 'git_repository', '*')) # The section below seems useless since we override buildout
assert len(repository_path_list) >= 0 # configuration to use local checkouts
for repository_path in repository_path_list: #repository_path_list = glob(os.path.join(config['software_root'],
updater = Updater(repository_path, git_binary=config['git_binary']) #'*', 'parts', 'git_repository', '*'))
updater.checkout() #assert len(repository_path_list) >= 0
if os.path.split(repository_path)[-1] == repository_name: #for repository_path in repository_path_list:
# redo checkout with good revision, the previous one is used #updater = Updater(repository_path, git_binary=config['git_binary'])
# to pull last code #updater.checkout()
updater = Updater(repository_path, git_binary=config['git_binary'], #if os.path.split(repository_path)[-1] == repository_name:
revision=revision) ## redo checkout with good revision, the previous one is used
updater.checkout() ## to pull last code
# calling dist/externals is only there for backward compatibility, #updater = Updater(repository_path, git_binary=config['git_binary'],
# the code will be removed soon #revision=revision)
if os.path.exists(os.path.join(repository_path, 'dist/externals.py')): #updater.checkout()
process = subprocess.Popen(['dist/externals.py'],
cwd=repository_path)
process.wait()
partition_path = os.path.join(config['instance_root'], partition_path = os.path.join(config['instance_root'],
config['partition_reference']) config['partition_reference'])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment