Commit 7a4f3c40 authored by Julien Muchembled's avatar Julien Muchembled

Merge upload_generic() into upload() and make it return the sha512

If people don't write stupid code like 'if result is True:',
this small API change should not break anything.

Add new index() method to add an entry to shadir when the data is known to be
already in shacache.
parent 7bb5e112
......@@ -25,8 +25,7 @@ import traceback
import urllib2
import urlparse
# XXX: code between select/select_generic and upload/upload_generic should be
# factored
# XXX: code between select/select_generic must be factored
# Timeout here is about timeout to CONNECT to the server (socket initialization then server answers actual data), not to retrieve/send informations.
# To be clear: it is NOT about uploading/downloading data, but about time to connect to the server, then time that server takes to start answering.
......@@ -165,9 +164,9 @@ class NetworkcacheClient(object):
self.shadir_cert_file = shadir_cert_file
def upload(self, file_descriptor, key=None, urlmd5=None, file_name=None,
valid_until=None, architecture=None):
valid_until=None, architecture=None, **kw):
''' Upload the file to the server.
If urlmd5 is None it must only upload to SHACACHE.
If key is None it must only upload to SHACACHE.
Otherwise, it must create a new entry on SHADIR.
'''
# do not trust, go to beginning of opened file
......@@ -182,12 +181,9 @@ class NetworkcacheClient(object):
self.shacache_port, timeout=UPLOAD_TIMEOUT)
try:
shacache_connection.request('POST', self.shacache_path, file_descriptor,
self.shacache_header_dict)
print 'uploaded'
self.shacache_header_dict)
result = shacache_connection.getresponse()
print 'answered'
sha512sum = result.read()
print 'read'
finally:
shacache_connection.close()
......@@ -197,104 +193,46 @@ class NetworkcacheClient(object):
% (self.shacache_host, result.status, sha512sum))
if key is not None:
kw['sha512'] = sha512sum # always update sha512sum
if file_name is None or urlmd5 is None:
raise ValueError('In case if key is given file_name and urlmd5 '
'are required.')
kw = dict()
kw['file'] = file_name
kw['urlmd5'] = urlmd5
kw['sha512'] = sha512sum
raise ValueError('file_name and urlmd5 are required'
' for non-generic upload')
if valid_until is not None:
kw['valid-until'] = valid_until
if architecture is not None:
kw['architecture'] = architecture
self.index(key, file=file_name, urlmd5=urlmd5, **kw)
sha_entry = json.dumps(kw)
try:
signature = self._getSignatureString(sha_entry)
except Exception:
raise UploadError('Impossible to sign content, error:\n%s' %
traceback.format_exc())
data = [sha_entry, signature]
if self.shadir_scheme == 'https':
shadir_connection = httplib.HTTPSConnection(self.shadir_host,
self.shadir_port, key_file=self.shadir_key_file,
cert_file=self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
else:
shadir_connection = httplib.HTTPConnection(self.shadir_host,
self.shadir_port, timeout=UPLOAD_TIMEOUT)
try:
shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
json.dumps(data), self.shadir_header_dict)
result = shadir_connection.getresponse()
data = result.read()
finally:
shadir_connection.close()
return sha512sum
if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.' \
'URL: %s. Response code: %s. Response data: %s' % \
(self.shacache_host, result.status, data))
return True
upload_generic = upload # BBB
def upload_generic(self, file_descriptor, key=None, **kw):
''' Upload the file to the server.
If key is None, it must only upload to SHACACHE.
Otherwise, it must create a new entry on SHADIR.
'''
file_descriptor.seek(0)
file_descriptor = hashing_file(file_descriptor)
if self.shacache_scheme == 'https':
shacache_connection = httplib.HTTPSConnection(self.shacache_host,
self.shacache_port, key_file = self.shacache_key_file,
cert_file = self.shacache_cert_file, timeout=UPLOAD_TIMEOUT)
def index(self, key, **kw):
data = json.dumps(kw)
try:
data = [data, self._getSignatureString(data)]
except Exception:
raise UploadError('Impossible to sign content, error:\n%s'
% traceback.format_exc())
if self.shadir_scheme == 'https':
shadir_connection = httplib.HTTPSConnection(self.shadir_host,
self.shadir_port, key_file=self.shadir_key_file,
cert_file=self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
else:
shacache_connection = httplib.HTTPConnection(self.shacache_host,
self.shacache_port, timeout=UPLOAD_TIMEOUT)
shadir_connection = httplib.HTTPConnection(self.shadir_host,
self.shadir_port, timeout=UPLOAD_TIMEOUT)
try:
shacache_connection.request('POST', self.shacache_path, file_descriptor,
self.shacache_header_dict)
result = shacache_connection.getresponse()
sha512sum = result.read()
shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
json.dumps(data), self.shadir_header_dict)
result = shadir_connection.getresponse()
data = result.read()
finally:
shacache_connection.close()
shadir_connection.close()
if result.status != 201 or sha512sum != file_descriptor.hexdigest():
raise UploadError('Failed to upload the file to SHACACHE Server.'
if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.'
'URL: %s. Response code: %s. Response data: %s'
% (self.shacache_host, result.status, sha512sum))
if key is not None:
kw['sha512'] = sha512sum # always update sha512sum
sha_entry = json.dumps(kw)
try:
signature = self._getSignatureString(sha_entry)
except Exception:
raise UploadError('Impossible to sign content, error:\n%s' %
traceback.format_exc())
data = [sha_entry, signature]
if self.shadir_scheme == 'https':
shadir_connection = httplib.HTTPSConnection(self.shadir_host,
self.shadir_port, key_file = self.shadir_key_file,
cert_file = self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
else:
shadir_connection = httplib.HTTPConnection(self.shadir_host,
self.shadir_port, timeout=UPLOAD_TIMEOUT)
try:
shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
json.dumps(data), self.shadir_header_dict)
result = shadir_connection.getresponse()
data = result.read()
finally:
shadir_connection.close()
if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.' \
'URL: %s. Response code: %s. Response data: %s' % \
(self.shadir_host, result.status, data))
return True
% (self.shadir_host, result.status, data))
def download(self, sha512sum):
''' Download the file.
......
......@@ -493,8 +493,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir, key_file.name, [self.certificate])
self.assertEqual(True, signed_nc.upload(self.test_data, key,
urlmd5=urlmd5, file_name=file_name))
self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
signed_nc.openssl = '/doesnotexists'
self.assertDirectoryNotFound('Could not find a trustable entry.',
signed_nc.select, key)
......@@ -508,8 +508,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir, key_file.name, [self.certificate])
self.assertEqual(True, signed_nc.upload(self.test_data, key,
urlmd5=urlmd5, file_name=file_name))
self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
signed_nc.openssl = sys.executable
self.assertDirectoryNotFound('Could not find a trustable entry.',
signed_nc.select, key)
......@@ -520,8 +520,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
file_name=file_name))
self.assertEqual(self.test_shasum, nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
f = os.path.join(self.tree, 'shadir', key)
# now remove the entry from shacache
open(f, 'w').write(json.dumps([]))
......@@ -534,8 +534,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
file_name=file_name))
self.assertEqual(self.test_shasum, nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache
f.write('This is not a json.')
......@@ -548,8 +548,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
file_name=file_name))
self.assertEqual(self.test_shasum, nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache
f.write(json.dumps([['This is not a json.', 'signature']]))
......@@ -563,8 +563,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
file_name=file_name))
self.assertEqual(self.test_shasum, nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache
f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))
......
......@@ -53,10 +53,8 @@ def helper_upload_network_cached(dir_url, cache_url,
return False
# backward compatibility
if not metadata_dict.get('file'):
metadata_dict['file'] = 'notused'
if not metadata_dict.get('urlmd5'):
metadata_dict['urlmd5'] = 'notused'
metadata_dict.setdefault('file', 'notused')
metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely
if not signature_private_key_file:
......@@ -81,7 +79,7 @@ def helper_upload_network_cached(dir_url, cache_url,
return False
try:
return nc.upload_generic(file_descriptor, directory_key, **metadata_dict)
return nc.upload(file_descriptor, directory_key, **metadata_dict)
except (IOError, UploadError), e:
logger.info('Failed to upload file. %s' % str(e))
return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment