Add strategy parameter, allowing to choose best entry.

parent dee03eda
...@@ -129,27 +129,32 @@ def helper_upload_network_cached_from_directory(dir_url, cache_url, ...@@ -129,27 +129,32 @@ def helper_upload_network_cached_from_directory(dir_url, cache_url,
signature_private_key_file, shacache_cert_file, shacache_key_file, signature_private_key_file, shacache_cert_file, shacache_key_file,
shadir_cert_file, shadir_key_file) shadir_cert_file, shadir_key_file)
finally: finally:
# Clean it # Always clean it
if os.path.exists(tarpath):
os.remove(tarpath) os.remove(tarpath)
return result return result
def helper_download_network_cached(dir_url, cache_url, def helper_download_network_cached(dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
directory_key, wanted_metadata_dict={}, required_key_list=[]): directory_key, wanted_metadata_dict={}, required_key_list=[],
strategy=None):
""" """
Downloads from a network cache provider. Downloads from a network cache provider.
Select from shadir directory_key entry matching (at least) Select from shadir directory_key entry matching (at least)
wanted_metadata_dict and with all metadata keys in required_key_list defined wanted_metadata_dict and with all metadata keys in required_key_list defined
and not null. and not null.
if a "strategy" function is given as parameter, use it to choose the best
entry between list of matching entries. Otherwise, choose the first.
This strategy function takes a list of entries as parameter, and should
return the best entry.
If something fails (providor be offline, or hash_string fail), we ignore If something fails (providor be offline, or hash_string fail), we ignore
network cached index. network cached index.
return (file_descriptor, metadata) if succeeded, False otherwise. return (file_descriptor, metadata) if succeeded, False otherwise.
""" """
# XXX It should be possible to give a function as parameter to specify
# strategy to choose best matching entry.
if not(dir_url and cache_url): if not(dir_url and cache_url):
return False return False
...@@ -170,6 +175,7 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -170,6 +175,7 @@ def helper_download_network_cached(dir_url, cache_url,
json_entry_list = nc.select_generic(directory_key) json_entry_list = nc.select_generic(directory_key)
# For each entry shadir sent, chooses only the entry matching all # For each entry shadir sent, chooses only the entry matching all
# wanted metadata, and having all wanted keys # wanted metadata, and having all wanted keys
matching_entry_list = []
for entry, _ in json_entry_list: for entry, _ in json_entry_list:
try: try:
tags = json.loads(entry) tags = json.loads(entry)
...@@ -188,11 +194,23 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -188,11 +194,23 @@ def helper_download_network_cached(dir_url, cache_url,
break break
if not match: if not match:
continue continue
# Everything match. Downloads corresponding file # Everything match. Add it to list of matching entries
file_descriptor = nc.download(tags.get('sha512')) matching_entry_list.append(tags)
break
except Exception: except Exception:
pass pass
# If a strategy is defined, call it to determine best entry
if strategy:
best_entry = strategy(matching_entry_list)
if not best_entry:
logger.info("Can't find best entry matching strategy, selecting "
"random one between acceptable ones.")
best_entry = matching_entry_list[0]
else:
best_entry = matching_entry_list[0]
# download best entry
file_descriptor = nc.download(best_entry.get('sha512'))
if file_descriptor is not None: if file_descriptor is not None:
return file_descriptor, tags return file_descriptor, tags
else: else:
...@@ -211,14 +229,15 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -211,14 +229,15 @@ def helper_download_network_cached(dir_url, cache_url,
def helper_download_network_cached_to_file(dir_url, cache_url, def helper_download_network_cached_to_file(dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
directory_key, path, wanted_metadata_dict={}): directory_key, path, wanted_metadata_dict={}, required_key_list=[],
strategy=None):
""" """
Download a file from network cache. It is the responsibility of caller method Download a file from network cache. It is the responsibility of caller method
to check md5. to check md5.
""" """
result = helper_download_network_cached(dir_url, cache_url, result = helper_download_network_cached(dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
directory_key, wanted_metadata_dict) directory_key, wanted_metadata_dict, required_key_list, strategy)
if result: if result:
# XXX check if nc filters signature_certificate_list! # XXX check if nc filters signature_certificate_list!
# Creates a file with content to desired path. # Creates a file with content to desired path.
...@@ -235,7 +254,8 @@ def helper_download_network_cached_to_file(dir_url, cache_url, ...@@ -235,7 +254,8 @@ def helper_download_network_cached_to_file(dir_url, cache_url,
def helper_download_network_cached_to_directory(dir_url, cache_url, def helper_download_network_cached_to_directory(dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
directory_key, path, wanted_metadata_dict={}): directory_key, path, wanted_metadata_dict={}, required_key_list=[],
strategy=None):
""" """
Download a tar file from network cache and untar it to specified path. Download a tar file from network cache and untar it to specified path.
""" """
...@@ -246,7 +266,8 @@ def helper_download_network_cached_to_directory(dir_url, cache_url, ...@@ -246,7 +266,8 @@ def helper_download_network_cached_to_directory(dir_url, cache_url,
metadata_dict = helper_download_network_cached_to_file( metadata_dict = helper_download_network_cached_to_file(
dir_url, cache_url, dir_url, cache_url,
signature_certificate_list, signature_certificate_list,
directory_key, tarpath, wanted_metadata_dict) directory_key, tarpath, wanted_metadata_dict, required_key_list,
strategy)
if metadata_dict: if metadata_dict:
# Untar it to path # Untar it to path
tar = tarfile.open(tarpath) tar = tarfile.open(tarpath)
...@@ -258,5 +279,6 @@ def helper_download_network_cached_to_directory(dir_url, cache_url, ...@@ -258,5 +279,6 @@ def helper_download_network_cached_to_directory(dir_url, cache_url,
finally: finally:
# Always clean it # Always clean it
if os.path.exists(tarpath):
os.remove(tarpath) os.remove(tarpath)
return metadata_dict return metadata_dict
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment