Commit 24f92f40 authored by Łukasz Nowak's avatar Łukasz Nowak

Minimise memory footprint during upload.

There is no need to read whole file into memory on library level. So calculate
sum in chunks and give request method file descriptor.
parent 81e88815
...@@ -68,14 +68,21 @@ class NetworkcacheClient(object): ...@@ -68,14 +68,21 @@ class NetworkcacheClient(object):
msg = 'The parameter "urlmd5" is required once you set directory_key.' msg = 'The parameter "urlmd5" is required once you set directory_key.'
raise ValueError(msg) raise ValueError(msg)
file_content = file_descriptor.read() sha512sum = hashlib.sha512()
sha512sum = hashlib.sha512(file_content).hexdigest() # do not trust, go to beginning of opened file
path = os.path.join(self.shacache_path, sha512sum) file_descriptor.seek(0)
while True:
d = file_descriptor.read(sha512sum.block_size)
if not d:
break
sha512sum.update(d)
path = os.path.join(self.shacache_path, sha512sum.hexdigest())
file_descriptor.seek(0)
shacache_connection = httplib.HTTPConnection(self.shacache_host, shacache_connection = httplib.HTTPConnection(self.shacache_host,
self.shacache_port) self.shacache_port)
try: try:
shacache_connection.request('PUT', path, file_content, shacache_connection.request('PUT', path, file_descriptor,
self.shacache_header_dict) self.shacache_header_dict)
result = shacache_connection.getresponse() result = shacache_connection.getresponse()
data = result.read() data = result.read()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment