Commit 08aa5cab authored by Lucas Carvalho's avatar Lucas Carvalho

The directory key can not be always the same.

So, a new method has been implemented in order to define the directory
key for each file which the buildout must download.
parent 7becb258
...@@ -20,7 +20,8 @@ import re ...@@ -20,7 +20,8 @@ import re
import urllib import urllib
import urlparse import urlparse
from download import check_md5sum from download import check_md5sum
from slapos.libnetworkcache import NetworkcacheClient, UploadError from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
MultipleFileFoundError, DirectoryNotFound
_md5_re = re.compile(r'md5=([a-f0-9]+)') _md5_re = re.compile(r'md5=([a-f0-9]+)')
...@@ -33,18 +34,26 @@ def _get_md5_from_url(url): ...@@ -33,18 +34,26 @@ def _get_md5_from_url(url):
return None return None
def get_directory_key(url):
"""It must retorn the directory key based on the URL.
Basically check if the url belongs to pypi:
- if yes, the directory key will be pypi-buildout-urlmd5
- if not, the directory key will be slapos-buildout-urlmd5
"""
urlmd5 = hashlib.md5(url).hexdigest()
if 'pypi' in url:
return 'pypi-buildout-%s' % urlmd5
return 'slapos-buildout-%s' % urlmd5
def select_sha512sum_from_shadir(sha_dir, url, logger): def select_sha512sum_from_shadir(sha_dir, url, logger):
"""It must return the sha512sum if exists on selected directory. """ """It must return the sha512sum if exists on selected directory. """
# On buildout.cfg the option sha-dir is the complete link including the directory_key = get_directory_key(url)
# directory key, so we must to extract it to use the libnetworkcache
# correctly.
directory_key = sha_dir.split('/')[-1]
sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
try: try:
nc = NetworkcacheClient(shadir=sha_dir_without_directory_key) nc = NetworkcacheClient(shadir=sha_dir)
data_list = nc.select(directory_key) data_list = nc.select(directory_key)
except (IOError, UploadError), e: except (IOError, DirectoryNotFound, MultipleFileFoundError), e:
logger.info('Could not retrieve the information from SHADIR. %s' % e) logger.info('Could not retrieve the information from SHADIR. %s' % e)
return None return None
...@@ -98,20 +107,14 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger): ...@@ -98,20 +107,14 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
'Upload cache ignored, shacache or shadir was not provided!') 'Upload cache ignored, shacache or shadir was not provided!')
return False return False
# On buildout.cfg the option sha-dir is the complete link including the directory_key = get_directory_key(external_url)
# directory key, so we must to extract it to use the libnetworkcache kw = dict(file_name=os.path.basename(external_url),
# correctly. urlmd5=hashlib.md5(external_url).hexdigest())
directory_key = sha_dir.split('/')[-1]
sha_dir_without_directory_key = '/'.join(sha_dir.split('/')[:-1])
f = open(path, 'r') f = open(path, 'r')
try: try:
kw = dict(file_name=os.path.basename(external_url),
urlmd5=hashlib.md5(external_url).hexdigest())
nc = NetworkcacheClient(shacache=sha_cache, nc = NetworkcacheClient(shacache=sha_cache,
shadir=sha_dir_without_directory_key) shadir=sha_dir)
data = nc.upload(f, directory_key, **kw) data = nc.upload(f, directory_key, **kw)
except (IOError, UploadError), e: except (IOError, UploadError), e:
logger.info('Fail to upload file. %s' % \ logger.info('Fail to upload file. %s' % \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment