.

nxdbom/__init__.py

@@ -39,6 +39,11 @@ from urllib.request import urlretrieve
 import email.parser
 import zipfile
 import shutil
+from contextlib import closing
+from hashlib import md5
+import logging
+
+from slapos.libnetworkcache import NetworkcacheClient
 
 
 # PkgInfo represents information about a package
@@ -557,14 +562,43 @@ def license_of(url):
         pkgfile = '.CACHE/pkg/' + basename(xpath)
         if not exists(pkgfile):
             mkdir_p('.CACHE/pkg')
-            print('DL %s ...' % url,  file=sys.stderr)
-            urlretrieve(url, pkgfile)
-        
+            wget(url, pkgfile)
+
         #shutil.unpack_archive(pkgfile, xpath)
 
     # XXX
     return None
 
+
+# wget downloads url to dstfile.
+# it first tries shacache and the direct download.
+_nc = None
+def wget(url, dstfile):
+    # see src/zc/buildout/download.py
+    global _nc
+    if _nc is None:
+        _nc = NetworkcacheClient({
+                    'download-cache-url': 'http://shacache.nxdcdn.com',
+                    'download-dir-url':   'http://shadir.nxdcdn.com',
+        })
+    cachekey = 'file-urlmd5:' + md5(url.encode('utf-8')).hexdigest()
+    if _nc.tryDownload(cachekey):
+        with _nc:
+            entry = next(_nc.select(cachekey, {'url': url}), None)
+            if entry is not None:
+                #print('DL %s ... (cache)' % url,  file=sys.stderr)
+                with closing(_nc.download(entry['sha512'])) as src, \
+                     open(dstfile+'.part', 'wb') as dst:
+                    shutil.copyfileobj(src, dst)
+                os.rename(dstfile+'.part', dstfile)
+                return
+
+    # direct
+    print('DL %s ... (direct)' % url,  file=sys.stderr)
+    1/0
+    #urlretrieve(url, dstfile)
+
+
 # XXX temp hack: disable ipv6
 # https://stackoverflow.com/a/6319043/9456786
 import socket
@@ -610,6 +644,8 @@ def main():
         print(__doc__, file=sys.stderr)
         sys.exit(2)
 
+    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
     what, arg = sys.argv[1:]
     if what == 'software':
         bom = bom_software(arg)

setup.py

@@ -13,6 +13,7 @@ setup(
     keywords    = 'Nexedi software build BOM',
 
     packages    = find_packages(),
+    install_requires = ['slapos.libnetworkcache'],
     extras_require = {
                    'test': ['pytest'],
     },

x.sh

 #!/bin/bash
 
-python3.9 nxdbom/__init__.py software ../runner/software/7f1663e8148f227ce3c6a38fc52796e2	# erp5
+nxdbom software ../runner/software/7f1663e8148f227ce3c6a38fc52796e2	# erp5