From 6f21e379483c60726a46323c660043dc4f825d2d Mon Sep 17 00:00:00 2001 From: Yusei Tahara Date: Tue, 16 Apr 2019 15:55:54 +0900 Subject: [PATCH] zc/buildout/easy_install.py: Buildout did not use find-links properly because of networkcache. Fix this problem. --- src/zc/buildout/easy_install.py | 80 ++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/src/zc/buildout/easy_install.py b/src/zc/buildout/easy_install.py index b13a40d7..b5cfe3b1 100644 --- a/src/zc/buildout/easy_install.py +++ b/src/zc/buildout/easy_install.py @@ -128,6 +128,7 @@ class _NoWarn(object): _no_warn = _NoWarn() +to_scan_page_cache = {} class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): """Will allow urls that are local to the system. @@ -143,14 +144,45 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): return setuptools.package_index.PackageIndex.url_ok( self, url, False) - def obtain(self, requirement, installer=None): + def obtain(self, requirement, installer=None, source=None): + """Our strategy is: + 1. Use the default index. Mostly it is enough. + 2. If a requirement was not met, then use find-links one by one. + A new parameter `source` was added to find a source distribution + for slapos.rebootstrap. + """ self._current_requirement = str(requirement) - return super(AllowHostsPackageIndex, self).obtain(requirement, installer) + self.find_packages(requirement) + for dist in self[requirement.key]: + if dist in requirement: + if source is not None: + if dist.precedence == pkg_resources.SOURCE_DIST: + return dist + else: + return dist + self.debug("%s does not match %s", requirement, dist) + + for find_link in self.to_scan: + self.scan_url(find_link) + for dist in self[requirement.key]: + if dist in requirement: + if source is not None: + if dist.precedence == pkg_resources.SOURCE_DIST: + return dist + else: + return dist + self.debug("%s does not match %s", requirement, dist) + + return super(PackageIndex, self).obtain(requirement, installer) def find_packages(self, requirement): self._current_requirement = str(requirement) return super(AllowHostsPackageIndex, self).find_packages(requirement) + def prescan(self): + """We do not use prescan because it does not make sense with networkcache.""" + pass + def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" if url in self.scanned_urls and not retrieve: @@ -166,7 +198,7 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): return self.debug("Found link: %s", url) - if dists or not retrieve or url in self.fetched_urls: + if dists or not retrieve or (not url in self.to_scan and url in self.fetched_urls): list(map(self.add, dists)) return # don't need the actual page @@ -182,6 +214,7 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): # If there is specific version, we want index to freeze forever. download_result = False requirement = getattr(self, '_current_requirement', '') + requirement_name = requirement.split('==')[0] from .buildout import network_cache_parameter_dict as nc if '==' in requirement: download_result = download_index_network_cached( @@ -194,12 +227,39 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): page, base = download_result f = None self.fetched_urls[base] = True + elif url in to_scan_page_cache: + page_cache = to_scan_page_cache[url] + if page_cache is None: + return + page, base = page_cache + if (requirement_name + and requirement_name in page + and nc.get('upload-cache-url') + and nc.get('upload-dir-url') + and '==' in requirement): + upload_index_network_cached( + nc.get('upload-dir-url'), + nc.get('upload-cache-url'), + url, base, requirement, page, logger, + nc.get('signature-private-key-file'), + nc.get('shacache-ca-file'), + nc.get('shacache-cert-file'), + nc.get('shacache-key-file'), + nc.get('shadir-ca-file'), + nc.get('shadir-cert-file'), + nc.get('shadir-key-file')) + return else: f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url) - if f is None: return + if f is None: + if url in self.to_scan: + to_scan_page_cache[url] = None + return self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it + if url in self.to_scan: + to_scan_page_cache[url] = None return base = f.url # handle redirects @@ -212,8 +272,16 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex): charset = f.headers.get_param('charset') or 'latin-1' page = page.decode(charset, "ignore") f.close() + + if url in self.to_scan: + if getattr(f, 'code', None) != 404: + to_scan_page_cache[url] = page, base + else: + to_scan_page_cache[url] = None + # Upload the index to network cache. - if nc.get('upload-cache-url') \ + if (not url in self.to_scan or (requirement_name and requirement_name in page)) \ + and nc.get('upload-cache-url') \ and nc.get('upload-dir-url') \ and '==' in requirement \ and getattr(f, 'code', None) != 404: @@ -541,7 +609,7 @@ class Installer: # initialize out index for this project: index = self._index - if index.obtain(requirement) is None: + if index.obtain(requirement, source=source) is None: # Nothing is available. return None -- 2.30.9