Commit 7a4f3c40 authored by Julien Muchembled's avatar Julien Muchembled

Merge upload_generic() into upload() and make it return the sha512

If people don't write stupid code like 'if result is True:',
this small API change should not break anything.

Add new index() method to add an entry to shadir when the data is known to be
already in shacache.
parent 7bb5e112
...@@ -25,8 +25,7 @@ import traceback ...@@ -25,8 +25,7 @@ import traceback
import urllib2 import urllib2
import urlparse import urlparse
# XXX: code between select/select_generic and upload/upload_generic should be # XXX: code between select/select_generic must be factored
# factored
# Timeout here is about timeout to CONNECT to the server (socket initialization then server answers actual data), not to retrieve/send informations. # Timeout here is about timeout to CONNECT to the server (socket initialization then server answers actual data), not to retrieve/send informations.
# To be clear: it is NOT about uploading/downloading data, but about time to connect to the server, then time that server takes to start answering. # To be clear: it is NOT about uploading/downloading data, but about time to connect to the server, then time that server takes to start answering.
...@@ -165,9 +164,9 @@ class NetworkcacheClient(object): ...@@ -165,9 +164,9 @@ class NetworkcacheClient(object):
self.shadir_cert_file = shadir_cert_file self.shadir_cert_file = shadir_cert_file
def upload(self, file_descriptor, key=None, urlmd5=None, file_name=None, def upload(self, file_descriptor, key=None, urlmd5=None, file_name=None,
valid_until=None, architecture=None): valid_until=None, architecture=None, **kw):
''' Upload the file to the server. ''' Upload the file to the server.
If urlmd5 is None it must only upload to SHACACHE. If key is None it must only upload to SHACACHE.
Otherwise, it must create a new entry on SHADIR. Otherwise, it must create a new entry on SHADIR.
''' '''
# do not trust, go to beginning of opened file # do not trust, go to beginning of opened file
...@@ -183,11 +182,8 @@ class NetworkcacheClient(object): ...@@ -183,11 +182,8 @@ class NetworkcacheClient(object):
try: try:
shacache_connection.request('POST', self.shacache_path, file_descriptor, shacache_connection.request('POST', self.shacache_path, file_descriptor,
self.shacache_header_dict) self.shacache_header_dict)
print 'uploaded'
result = shacache_connection.getresponse() result = shacache_connection.getresponse()
print 'answered'
sha512sum = result.read() sha512sum = result.read()
print 'read'
finally: finally:
shacache_connection.close() shacache_connection.close()
...@@ -197,27 +193,27 @@ class NetworkcacheClient(object): ...@@ -197,27 +193,27 @@ class NetworkcacheClient(object):
% (self.shacache_host, result.status, sha512sum)) % (self.shacache_host, result.status, sha512sum))
if key is not None: if key is not None:
kw['sha512'] = sha512sum # always update sha512sum
if file_name is None or urlmd5 is None: if file_name is None or urlmd5 is None:
raise ValueError('In case if key is given file_name and urlmd5 ' raise ValueError('file_name and urlmd5 are required'
'are required.') ' for non-generic upload')
kw = dict()
kw['file'] = file_name
kw['urlmd5'] = urlmd5
kw['sha512'] = sha512sum
if valid_until is not None: if valid_until is not None:
kw['valid-until'] = valid_until kw['valid-until'] = valid_until
if architecture is not None: if architecture is not None:
kw['architecture'] = architecture kw['architecture'] = architecture
self.index(key, file=file_name, urlmd5=urlmd5, **kw)
return sha512sum
upload_generic = upload # BBB
sha_entry = json.dumps(kw) def index(self, key, **kw):
data = json.dumps(kw)
try: try:
signature = self._getSignatureString(sha_entry) data = [data, self._getSignatureString(data)]
except Exception: except Exception:
raise UploadError('Impossible to sign content, error:\n%s' % raise UploadError('Impossible to sign content, error:\n%s'
traceback.format_exc()) % traceback.format_exc())
data = [sha_entry, signature]
if self.shadir_scheme == 'https': if self.shadir_scheme == 'https':
shadir_connection = httplib.HTTPSConnection(self.shadir_host, shadir_connection = httplib.HTTPSConnection(self.shadir_host,
self.shadir_port, key_file=self.shadir_key_file, self.shadir_port, key_file=self.shadir_key_file,
...@@ -234,67 +230,9 @@ class NetworkcacheClient(object): ...@@ -234,67 +230,9 @@ class NetworkcacheClient(object):
shadir_connection.close() shadir_connection.close()
if result.status != 201: if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.' \ raise UploadError('Failed to upload data to SHADIR Server.'
'URL: %s. Response code: %s. Response data: %s' % \
(self.shacache_host, result.status, data))
return True
def upload_generic(self, file_descriptor, key=None, **kw):
''' Upload the file to the server.
If key is None, it must only upload to SHACACHE.
Otherwise, it must create a new entry on SHADIR.
'''
file_descriptor.seek(0)
file_descriptor = hashing_file(file_descriptor)
if self.shacache_scheme == 'https':
shacache_connection = httplib.HTTPSConnection(self.shacache_host,
self.shacache_port, key_file = self.shacache_key_file,
cert_file = self.shacache_cert_file, timeout=UPLOAD_TIMEOUT)
else:
shacache_connection = httplib.HTTPConnection(self.shacache_host,
self.shacache_port, timeout=UPLOAD_TIMEOUT)
try:
shacache_connection.request('POST', self.shacache_path, file_descriptor,
self.shacache_header_dict)
result = shacache_connection.getresponse()
sha512sum = result.read()
finally:
shacache_connection.close()
if result.status != 201 or sha512sum != file_descriptor.hexdigest():
raise UploadError('Failed to upload the file to SHACACHE Server.'
'URL: %s. Response code: %s. Response data: %s' 'URL: %s. Response code: %s. Response data: %s'
% (self.shacache_host, result.status, sha512sum)) % (self.shadir_host, result.status, data))
if key is not None:
kw['sha512'] = sha512sum # always update sha512sum
sha_entry = json.dumps(kw)
try:
signature = self._getSignatureString(sha_entry)
except Exception:
raise UploadError('Impossible to sign content, error:\n%s' %
traceback.format_exc())
data = [sha_entry, signature]
if self.shadir_scheme == 'https':
shadir_connection = httplib.HTTPSConnection(self.shadir_host,
self.shadir_port, key_file = self.shadir_key_file,
cert_file = self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
else:
shadir_connection = httplib.HTTPConnection(self.shadir_host,
self.shadir_port, timeout=UPLOAD_TIMEOUT)
try:
shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
json.dumps(data), self.shadir_header_dict)
result = shadir_connection.getresponse()
data = result.read()
finally:
shadir_connection.close()
if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.' \
'URL: %s. Response code: %s. Response data: %s' % \
(self.shadir_host, result.status, data))
return True
def download(self, sha512sum): def download(self, sha512sum):
''' Download the file. ''' Download the file.
......
...@@ -493,8 +493,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -493,8 +493,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key_file.flush() key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient( signed_nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir, key_file.name, [self.certificate]) self.shacache, self.shadir, key_file.name, [self.certificate])
self.assertEqual(True, signed_nc.upload(self.test_data, key, self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
urlmd5=urlmd5, file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
signed_nc.openssl = '/doesnotexists' signed_nc.openssl = '/doesnotexists'
self.assertDirectoryNotFound('Could not find a trustable entry.', self.assertDirectoryNotFound('Could not find a trustable entry.',
signed_nc.select, key) signed_nc.select, key)
...@@ -508,8 +508,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -508,8 +508,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key_file.flush() key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient( signed_nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir, key_file.name, [self.certificate]) self.shacache, self.shadir, key_file.name, [self.certificate])
self.assertEqual(True, signed_nc.upload(self.test_data, key, self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
urlmd5=urlmd5, file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
signed_nc.openssl = sys.executable signed_nc.openssl = sys.executable
self.assertDirectoryNotFound('Could not find a trustable entry.', self.assertDirectoryNotFound('Could not find a trustable entry.',
signed_nc.select, key) signed_nc.select, key)
...@@ -520,8 +520,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -520,8 +520,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file' file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient( nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir) self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5, self.assertEqual(self.test_shasum, nc.upload(self.test_data,
file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
f = os.path.join(self.tree, 'shadir', key) f = os.path.join(self.tree, 'shadir', key)
# now remove the entry from shacache # now remove the entry from shacache
open(f, 'w').write(json.dumps([])) open(f, 'w').write(json.dumps([]))
...@@ -534,8 +534,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -534,8 +534,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file' file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient( nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir) self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5, self.assertEqual(self.test_shasum, nc.upload(self.test_data,
file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write('This is not a json.') f.write('This is not a json.')
...@@ -548,8 +548,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -548,8 +548,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file' file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient( nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir) self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5, self.assertEqual(self.test_shasum, nc.upload(self.test_data,
file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write(json.dumps([['This is not a json.', 'signature']])) f.write(json.dumps([['This is not a json.', 'signature']]))
...@@ -563,8 +563,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -563,8 +563,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
file_name = 'my file' file_name = 'my file'
nc = slapos.libnetworkcache.NetworkcacheClient( nc = slapos.libnetworkcache.NetworkcacheClient(
self.shacache, self.shadir) self.shacache, self.shadir)
self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5, self.assertEqual(self.test_shasum, nc.upload(self.test_data,
file_name=file_name)) key, urlmd5=urlmd5, file_name=file_name))
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write(json.dumps([[json.dumps('This is a string'), 'signature']])) f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))
......
...@@ -53,10 +53,8 @@ def helper_upload_network_cached(dir_url, cache_url, ...@@ -53,10 +53,8 @@ def helper_upload_network_cached(dir_url, cache_url,
return False return False
# backward compatibility # backward compatibility
if not metadata_dict.get('file'): metadata_dict.setdefault('file', 'notused')
metadata_dict['file'] = 'notused' metadata_dict.setdefault('urlmd5', 'notused')
if not metadata_dict.get('urlmd5'):
metadata_dict['urlmd5'] = 'notused'
# convert '' into None in order to call nc nicely # convert '' into None in order to call nc nicely
if not signature_private_key_file: if not signature_private_key_file:
...@@ -81,7 +79,7 @@ def helper_upload_network_cached(dir_url, cache_url, ...@@ -81,7 +79,7 @@ def helper_upload_network_cached(dir_url, cache_url,
return False return False
try: try:
return nc.upload_generic(file_descriptor, directory_key, **metadata_dict) return nc.upload(file_descriptor, directory_key, **metadata_dict)
except (IOError, UploadError), e: except (IOError, UploadError), e:
logger.info('Failed to upload file. %s' % str(e)) logger.info('Failed to upload file. %s' % str(e))
return False return False
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment