easy_install.py 79.9 KB
Newer Older
1
#############################################################################
2
#
icemac's avatar
icemac committed
3
# Copyright (c) 2005 Zope Foundation and Contributors.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python easy_install API

This module provides a high-level Python API for installing packages.
It doesn't install scripts.  It uses setuptools and requires it to be
installed.
"""

21
import distutils.errors
22
import fnmatch
23 24 25
import glob
import logging
import os
26
import pkg_resources
27 28 29
import py_compile
import re
import setuptools.archive_util
30 31
import setuptools.command.setopt
import setuptools.package_index
32
import shutil
33
import subprocess
34 35
import sys
import tempfile
36
import warnings
37
import zc.buildout
38
import zipimport
39

40
_oprp = getattr(os.path, 'realpath', lambda path: path)
41
def realpath(path):
42
    return os.path.normcase(os.path.abspath(_oprp(path)))
43

44 45 46

from zc.buildout.networkcache import get_filename_from_url, \
                                     upload_network_cached, \
47 48 49
                                     download_network_cached, \
                                     download_index_network_cached, \
                                     upload_index_network_cached
50 51


52 53
default_index_url = os.environ.get(
    'buildout-testing-index-url',
54
    'http://pypi.python.org/simple',
55
    )
56

57 58
logger = logging.getLogger('zc.buildout.easy_install')

59 60
url_match = re.compile('[a-z0-9+.-]+://').match

61
is_win32 = sys.platform == 'win32'
georgyberdyshev's avatar
georgyberdyshev committed
62
is_jython = sys.platform.startswith('java')
63 64
is_distribute = (
    pkg_resources.Requirement.parse('setuptools').key=='distribute')
georgyberdyshev's avatar
georgyberdyshev committed
65

66 67 68 69 70 71 72 73 74 75 76 77
BROKEN_DASH_S_WARNING = (
    'Buildout has been asked to exclude or limit site-packages so that '
    'builds can be repeatable when using a system Python.  However, '
    'the chosen Python executable has a broken implementation of -S (see '
    'https://bugs.launchpad.net/virtualenv/+bug/572545 for an example '
    "problem) and this breaks buildout's ability to isolate site-packages.  "
    "If the executable already has a clean site-packages (e.g., "
    "using virtualenv's ``--no-site-packages`` option) you may be getting "
    'equivalent repeatability.  To silence this warning, use the -s argument '
    'to the buildout script.  Alternatively, use a Python executable with a '
    'working -S (such as a standard Python binary).')

georgyberdyshev's avatar
georgyberdyshev committed
78 79 80 81
if is_jython:
    import java.lang.System
    jython_os_name = (java.lang.System.getProperties()['os.name']).lower()

82 83 84 85
setuptools_loc = pkg_resources.working_set.find(
    pkg_resources.Requirement.parse('setuptools')
    ).location

86 87 88 89 90 91 92 93
# Include buildout and setuptools eggs in paths.  We prevent dupes just to
# keep from duplicating any log messages about them.
buildout_loc = pkg_resources.working_set.find(
    pkg_resources.Requirement.parse('zc.buildout')).location
buildout_and_setuptools_path = [setuptools_loc]
if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
    buildout_and_setuptools_path.append(buildout_loc)

94 95 96 97 98 99 100 101 102
def _has_broken_dash_S(executable):
    """Detect https://bugs.launchpad.net/virtualenv/+bug/572545 ."""
    # The first attempt here was to simply have the executable attempt to import
    # ConfigParser and return the return code. That worked except for tests on
    # Windows, where the return code was wrong for the fake Python executable
    # generated by the virtualenv.txt test, apparently because setuptools' .exe
    # file does not pass the -script.py's returncode back properly, at least in
    # some circumstances. Therefore...print statements.
    stdout, stderr = subprocess.Popen(
103
        [executable, '-S', '-c',
104 105 106 107 108 109 110 111 112
         'try:\n'
         '    import ConfigParser\n'
         'except ImportError:\n'
         '    print 1\n'
         'else:\n'
         '    print 0\n'],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    return bool(int(stdout.strip()))

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
def _get_system_paths(executable):
    """Return lists of standard lib and site paths for executable.
    """
    # We want to get a list of the site packages, which is not easy.
    # The canonical way to do this is to use
    # distutils.sysconfig.get_python_lib(), but that only returns a
    # single path, which does not reflect reality for many system
    # Pythons, which have multiple additions.  Instead, we start Python
    # with -S, which does not import site.py and set up the extra paths
    # like site-packages or (Ubuntu/Debian) dist-packages and
    # python-support. We then compare that sys.path with the normal one
    # (minus user packages if this is Python 2.6, because we don't
    # support those (yet?).  The set of the normal one minus the set of
    # the ones in ``python -S`` is the set of packages that are
    # effectively site-packages.
    #
    # The given executable might not be the current executable, so it is
    # appropriate to do another subprocess to figure out what the
    # additional site-package paths are. Moreover, even if this
    # executable *is* the current executable, this code might be run in
    # the context of code that has manipulated the sys.path--for
    # instance, to add local zc.buildout or setuptools eggs.
    def get_sys_path(*args, **kwargs):
        cmd = [executable]
        cmd.extend(args)
        cmd.extend([
            "-c", "import sys, os;"
            "print repr([os.path.normpath(p) for p in sys.path if p])"])
        # Windows needs some (as yet to be determined) part of the real env.
        env = os.environ.copy()
        # We need to make sure that PYTHONPATH, which will often be set
        # to include a custom buildout-generated site.py, is not set, or
        # else we will not get an accurate sys.path for the executable.
        env.pop('PYTHONPATH', None)
        env.update(kwargs)
        _proc = subprocess.Popen(
            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
        stdout, stderr = _proc.communicate();
        if _proc.returncode:
            raise RuntimeError(
                'error trying to get system packages:\n%s' % (stderr,))
        res = eval(stdout.strip())
        try:
            res.remove('.')
        except ValueError:
            pass
        return res
    stdlib = get_sys_path('-S') # stdlib only
    no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
    site_paths = [p for p in no_user_paths if p not in stdlib]
    return (stdlib, site_paths)

def _get_version_info(executable):
166 167
    cmd = [executable, '-Sc',
           'import sys; print(repr(tuple(x for x in sys.version_info)))']
168 169 170 171 172 173 174
    _proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = _proc.communicate();
    if _proc.returncode:
        raise RuntimeError(
            'error trying to get system packages:\n%s' % (stderr,))
    return eval(stdout.strip())
175

176

jim's avatar
jim committed
177 178 179 180
class IncompatibleVersionError(zc.buildout.UserError):
    """A specified version is incompatible with a given requirement.
    """

181 182 183 184 185
_versions = {sys.executable: '%d.%d' % sys.version_info[:2]}
def _get_version(executable):
    try:
        return _versions[executable]
    except KeyError:
186 187 188 189 190 191
        cmd = _safe_arg(executable) + ' -V'
        p = subprocess.Popen(cmd,
                             shell=True,
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
sidnei's avatar
sidnei committed
192
                             close_fds=not is_win32)
193
        i, o = (p.stdin, p.stdout)
194 195 196 197 198
        i.close()
        version = o.read().strip()
        o.close()
        pystring, version = version.split()
        assert pystring == 'Python'
199
        version = re.match('(\d[.]\d)([.].*\d)?$', version).group(1)
200 201 202
        _versions[executable] = version
        return version

tarek's avatar
tarek committed
203
FILE_SCHEME = re.compile('file://', re.I).match
204 205 206 207
URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
from setuptools.package_index import distros_for_url, htmldecode
import urlparse
208

tarek's avatar
tarek committed
209 210 211 212 213
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
    """Will allow urls that are local to the system.

    No matter what is allow_hosts.
    """
214 215 216 217 218 219

    def __init__(self, index_url, hosts, search_path, python,):
        setuptools.package_index.PackageIndex.__init__(self,
           index_url, hosts=hosts, search_path=search_path,
           python=python)

tarek's avatar
tarek committed
220 221 222 223
    def url_ok(self, url, fatal=False):
        if FILE_SCHEME(url):
            return True
        return setuptools.package_index.PackageIndex.url_ok(self, url, False)
224

225 226 227 228 229 230 231 232 233 234
    def obtain(self, requirement, installer=None):
        # XXX BEGIN HARDCODE
        self._current_requirement = requirement.__str__()
        # XXX END HARDCODE

        self.prescan(); self.find_packages(requirement)
        for dist in self[requirement.key]:
            if dist in requirement:
                return dist
            self.debug("%s does not match %s", requirement, dist)
Marco Mariani's avatar
Marco Mariani committed
235
        return super(AllowHostsPackageIndex, self).obtain(requirement,installer)
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330

    def find_packages(self, requirement):
        # XXX BEGIN HARDCODE
        self._current_requirement = requirement.__str__()
        # XXX END HARDCODE

        self.scan_url(self.index_url + requirement.unsafe_name+'/')

        if not self.package_pages.get(requirement.key):
            # Fall back to safe version of the name
            self.scan_url(self.index_url + requirement.project_name+'/')

        if not self.package_pages.get(requirement.key):
            # We couldn't find the target package, so search the index page too
            self.not_found_in_index(requirement)

        for url in list(self.package_pages.get(requirement.key,())):
            # scan each page that might be related to the desired package
            self.scan_url(url)

    def process_url(self, url, retrieve=False):
        """Evaluate a URL as a possible download, and maybe retrieve it"""
        if url in self.scanned_urls and not retrieve:
            return
        self.scanned_urls[url] = True
        if not URL_SCHEME(url):
            self.process_filename(url)
            return
        else:
            dists = list(distros_for_url(url))
            if dists:
                if not self.url_ok(url):
                    return
                self.debug("Found link: %s", url)

        if dists or not retrieve or url in self.fetched_urls:
            map(self.add, dists)
            return  # don't need the actual page

        if not self.url_ok(url):
            self.fetched_urls[url] = True
            return

        self.info("Reading %s", url)
        self.fetched_urls[url] = True   # prevent multiple fetch attempts

        # We download from cache only if we have a specific version for egg requirement
        # Reason: If no specific version, we don't want to be blocked with an old index.
        #         If there is specific version, we want index to freeze forever.
        download_result = False
        if getattr(self, '_current_requirement', None) \
          and "==" in self._current_requirement \
          and globals().get('_download_dir_url') \
          and globals().get('_download_dir_url') \
          and globals().get('_signature_certificate_list'):
            download_result = download_index_network_cached(_download_dir_url,
               _download_cache_url, url, self._current_requirement, logger,
               _signature_certificate_list)
        if download_result:
            # Successfully fetched from cache. Hardcode some things...
            page, base = download_result
            f = None
            self.fetched_urls[base] = True
        else:
            f = self.open_url(url, "Download error: %s -- Some packages may not be found!")
            if f is None: return
            self.fetched_urls[f.url] = True
            if 'html' not in f.headers.get('content-type', '').lower():
                f.close()   # not html, we can't process it
                return

            base = f.url     # handle redirects
            page = f.read()
            f.close()

            # Check if shacache options exist in global vars, then check if
            # it is not null
            if globals().get('_upload_cache_url') \
               and globals().get('_upload_dir_url') \
               and getattr(self, '_current_requirement', None) \
               and "==" in self._current_requirement \
               and getattr(f,'code',None)!=404:
                upload_index_network_cached(_upload_dir_url,
                   _upload_cache_url, url, base, self._current_requirement, page, logger,
                   globals().get('_signature_private_key_file'),
                   globals().get('_shacache_cert_file'),
                   globals().get('_shacache_key_file'),
                   globals().get('_shadir_cert_file'),
                   globals().get('_shadir_key_file'))

        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
            page = self.process_index(url, page)
        for match in HREF.finditer(page):
            link = urlparse.urljoin(base, htmldecode(match.group(1)))
            self.process_url(link)
tarek's avatar
tarek committed
331

332
_indexes = {}
333
def _get_index(executable, index_url, find_links, allow_hosts=('*',),
334
               path=None,):
335 336 337 338
    # If path is None, the index will use sys.path.  If you provide an empty
    # path ([]), it will complain uselessly about missing index pages for
    # packages found in the paths that you expect to use.  Therefore, this path
    # is always the same as the _env path in the Installer.
339 340 341 342 343
    key = executable, index_url, tuple(find_links)
    index = _indexes.get(key)
    if index is not None:
        return index

344 345
    if index_url is None:
        index_url = default_index_url
tarek's avatar
tarek committed
346
    index = AllowHostsPackageIndex(
347
        index_url, hosts=allow_hosts, search_path=path,
348
        python=_get_version(executable),
349
        )
350

351 352 353 354 355 356
    if find_links:
        index.add_find_links(find_links)

    _indexes[key] = index
    return index

357
clear_index_cache = _indexes.clear
jim's avatar
jim committed
358

359
if is_win32:
jim's avatar
jim committed
360
    # work around spawn lamosity on windows
361
    # XXX need safe quoting (see the subprocess.list2cmdline) and test
jim's avatar
jim committed
362 363 364 365 366
    def _safe_arg(arg):
        return '"%s"' % arg
else:
    _safe_arg = str

367
# The following string is used to run easy_install in
368 369 370 371
# Installer._call_easy_install.  It is usually started with python -S
# (that is, don't import site at start).  That flag, and all of the code
# in this snippet above the last two lines, exist to work around a
# relatively rare problem.  If
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
#
# - your buildout configuration is trying to install a package that is within
#   a namespace package, and
#
# - you use a Python that has a different version of this package
#   installed in in its site-packages using
#   --single-version-externally-managed (that is, using the mechanism
#   sometimes used by system packagers:
#   http://peak.telecommunity.com/DevCenter/setuptools#install-command ), and
#
# - the new package tries to do sys.path tricks in the setup.py to get a
#   __version__,
#
# then the older package will be loaded first, making the setup version
# the wrong number. While very arguably packages simply shouldn't do
# the sys.path tricks, some do, and we don't want buildout to fall over
# when they do.
#
# The namespace packages installed in site-packages with
# --single-version-externally-managed use a mechanism that cause them to
# be processed when site.py is imported  (see
# http://mail.python.org/pipermail/distutils-sig/2009-May/011730.html
# for another description of the problem).  Simply starting Python with
# -S addresses the problem in Python 2.4 and 2.5, but Python 2.6's
# distutils imports a value from the site module, so we unfortunately
397
# have to do more drastic surgery in the _easy_install_preface code below.
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
#
# Here's an example of the .pth files created by setuptools when using that
# flag:
#
# import sys,new,os;
# p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('<NAMESPACE>',));
# ie = os.path.exists(os.path.join(p,'__init__.py'));
# m = not ie and sys.modules.setdefault('<NAMESPACE>',new.module('<NAMESPACE>'));
# mp = (m or []) and m.__dict__.setdefault('__path__',[]);
# (p not in mp) and mp.append(p)
#
# The code, below, then, runs under -S, indicating that site.py should
# not be loaded initially.  It gets the initial sys.path under these
# circumstances, and then imports site (because Python 2.6's distutils
# will want it, as mentioned above). It then reinstates the old sys.path
# value. Then it removes namespace packages (created by the setuptools
# code above) from sys.modules.  It identifies namespace packages by
# iterating over every loaded module.  It first looks if there is a
# __path__, so it is a package; and then it sees if that __path__ does
# not have an __init__.py.  (Note that PEP 382,
# http://www.python.org/dev/peps/pep-0382, makes it possible to have a
# namespace package that has an __init__.py, but also should make it
# unnecessary for site.py to preprocess these packages, so it should be
# fine, as far as can be guessed as of this writing.)  Finally, it
# imports easy_install and runs it.
423
_easy_install_preface = '''\
424 425 426 427 428 429
import sys,os;\
p = sys.path[:];\
import site;\
sys.path[:] = p;\
[sys.modules.pop(k) for k, v in sys.modules.items()\
 if hasattr(v, '__path__') and len(v.__path__)==1 and\
430 431 432
 not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];'''
_easy_install_cmd = (
    'from setuptools.command.easy_install import main;main()')
jim's avatar
jim committed
433

434

435 436
class Installer:

437
    _versions = {}
438
    _download_cache = None
439
    _install_from_cache = False
440
    _prefer_final = True
441
    _use_dependency_links = True
jim's avatar
jim committed
442
    _allow_picked_versions = True
443
    _always_unzip = False
444 445
    _include_site_packages = True
    _allowed_eggs_from_site_packages = ('*',)
446

447 448 449 450 451
    def __init__(self,
                 dest=None,
                 links=(),
                 index=None,
                 executable=sys.executable,
452
                 always_unzip=None,
453 454
                 path=None,
                 newest=True,
jim's avatar
jim committed
455
                 versions=None,
456
                 use_dependency_links=None,
457 458
                 allow_hosts=('*',),
                 include_site_packages=None,
459 460
                 allowed_eggs_from_site_packages=None,
                 prefer_final=None,
461
                 download_dir_url=None,
Łukasz Nowak's avatar
Łukasz Nowak committed
462 463
                 download_cache_url=None,
                 upload_dir_url=None,
464
                 upload_cache_url=None,
465
                 signature_certificate_list=None,
466 467 468 469 470
                 signature_private_key_file=None,
                 shacache_cert_file=None,
                 shacache_key_file=None,
                 shadir_cert_file=None,
                 shadir_key_file=None,
471
                 ):
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
        # XXX Setting shacahe options to global variables in order to be able
        # to use them from recipes not aware of it. I now hate myself.
        if download_dir_url:
            global _download_dir_url
            _download_dir_url = download_dir_url
        if download_cache_url:
            global _download_cache_url
            _download_cache_url = download_cache_url
        if upload_dir_url:
            global _upload_dir_url
            _upload_dir_url = upload_dir_url
        if upload_cache_url:
            global _upload_cache_url
            _upload_cache_url = upload_cache_url
        if signature_certificate_list:
            global _signature_certificate_list
            _signature_certificate_list = signature_certificate_list
        if signature_private_key_file:
            global _signature_private_key_file
            _signature_private_key_file = signature_private_key_file
        if shacache_cert_file:
            global _shacache_cert_file
            _shacache_cert_file = shacache_cert_file
        if shacache_key_file:
            global _shacache_key_file
            _shacache_key_file = shacache_key_file
        if shadir_cert_file:
            global _shadir_cert_file
            _shadir_cert_file = shadir_cert_file
        if shadir_key_file:
            global _shadir_key_file
            _shadir_key_file = shadir_key_file

505
        self._dest = dest
tarek's avatar
tarek committed
506
        self._allow_hosts = allow_hosts
507 508 509 510 511 512 513

        if self._install_from_cache:
            if not self._download_cache:
                raise ValueError("install_from_cache set to true with no"
                                 " download cache")
            links = ()
            index = 'file://' + self._download_cache
514

515 516
        if use_dependency_links is not None:
            self._use_dependency_links = use_dependency_links
517 518
        if prefer_final is not None:
            self._prefer_final = prefer_final
519
        self._links = links = list(_fix_file_links(links))
520 521 522
        if self._download_cache and (self._download_cache not in links):
            links.insert(0, self._download_cache)

523 524
        self._index_url = index
        self._executable = executable
525
        self._has_broken_dash_S = _has_broken_dash_S(self._executable)
526 527
        if always_unzip is not None:
            self._always_unzip = always_unzip
528 529 530 531 532 533
        path = (path and path[:] or [])
        if include_site_packages is not None:
            self._include_site_packages = include_site_packages
        if allowed_eggs_from_site_packages is not None:
            self._allowed_eggs_from_site_packages = tuple(
                allowed_eggs_from_site_packages)
534 535 536 537 538 539 540 541 542 543 544
        if self._has_broken_dash_S:
            if (not self._include_site_packages or
                self._allowed_eggs_from_site_packages != ('*',)):
                # We can't do this if the executable has a broken -S.
                warnings.warn(BROKEN_DASH_S_WARNING)
                self._include_site_packages = True
                self._allowed_eggs_from_site_packages = ('*',)
            self._easy_install_cmd = _easy_install_cmd
        else:
            self._easy_install_cmd = _easy_install_preface + _easy_install_cmd
        self._easy_install_cmd = _safe_arg(self._easy_install_cmd)
545 546 547 548 549 550 551 552 553 554 555 556 557
        stdlib, self._site_packages = _get_system_paths(executable)
        version_info = _get_version_info(executable)
        if version_info == sys.version_info:
            # Maybe we can add the buildout and setuptools path.  If we
            # are including site_packages, we only have to include the extra
            # bits here, so we don't duplicate.  On the other hand, if we
            # are not including site_packages, we only want to include the
            # parts that are not in site_packages, so the code is the same.
            path.extend(
                set(buildout_and_setuptools_path).difference(
                    self._site_packages))
        if self._include_site_packages:
            path.extend(self._site_packages)
558 559 560
        if dest is not None and dest not in path:
            path.insert(0, dest)
        self._path = path
561 562
        if self._dest is None:
            newest = False
563 564 565
        self._newest = newest
        self._env = pkg_resources.Environment(path,
                                              python=_get_version(executable))
566
        self._index = _get_index(executable, index, links, self._allow_hosts,
567
                                 self._path,)
568 569 570

        if versions is not None:
            self._versions = versions
571

572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
    _allowed_eggs_from_site_packages_regex = None
    def allow_site_package_egg(self, name):
        if (not self._include_site_packages or
            not self._allowed_eggs_from_site_packages):
            # If the answer is a blanket "no," perform a shortcut.
            return False
        if self._allowed_eggs_from_site_packages_regex is None:
            pattern = '(%s)' % (
                '|'.join(
                    fnmatch.translate(name)
                    for name in self._allowed_eggs_from_site_packages),
                )
            self._allowed_eggs_from_site_packages_regex = re.compile(pattern)
        return bool(self._allowed_eggs_from_site_packages_regex.match(name))

587
    def _satisfied(self, req, source=None):
588 589 590 591 592 593 594 595 596 597 598 599 600 601
        # We get all distributions that match the given requirement.  If we are
        # not supposed to include site-packages for the given egg, we also
        # filter those out. Even if include_site_packages is False and so we
        # have excluded site packages from the _env's paths (see
        # Installer.__init__), we need to do the filtering here because an
        # .egg-link, such as one for setuptools or zc.buildout installed by
        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
        # path in our _site_packages.
        dists = [dist for dist in self._env[req.project_name] if (
                    dist in req and (
                        dist.location not in self._site_packages or
                        self.allow_site_package_egg(dist.project_name))
                    )
                ]
602
        if not dists:
603 604
            logger.debug('We have no distributions for %s that satisfies %r.',
                         req.project_name, str(req))
605

606
            return None, self._obtain(req, source)
607 608 609 610 611

        # Note that dists are sorted from best to worst, as promised by
        # env.__getitem__

        for dist in dists:
612 613 614 615 616 617
            if (dist.precedence == pkg_resources.DEVELOP_DIST and
                dist.location not in self._site_packages):
                # System eggs are sometimes installed as develop eggs.
                # Those are not the kind of develop eggs we are looking for
                # here: we want ones that the buildout itself has locally as
                # develop eggs.
618
                logger.debug('We have a develop egg: %s', dist)
619
                return dist, None
620

621 622 623 624 625 626
        # Special common case, we have a specification for a single version:
        specs = req.specs
        if len(specs) == 1 and specs[0][0] == '==':
            logger.debug('We have the distribution that satisfies %r.',
                         str(req))
            return dists[0], None
627

628 629 630 631 632 633 634 635 636 637 638 639 640 641
        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        if not self._newest:
            # We don't need the newest, so we'll use the newest one we
            # find, which is the first returned by
            # Environment.__getitem__.
            return dists[0], None

642
        best_we_have = dists[0] # Because dists are sorted from best to worst
jim's avatar
jim committed
643

644 645 646 647
        # We have some installed distros.  There might, theoretically, be
        # newer ones.  Let's find out which ones are available and see if
        # any are newer.  We only do this if we're willing to install
        # something, which is only true if dest is not None:
648

649
        if self._dest is not None:
650
            best_available = self._obtain(req, source)
651 652 653 654 655 656 657
        else:
            best_available = None

        if best_available is None:
            # That's a bit odd.  There aren't any distros available.
            # We should use the best one we have that meets the requirement.
            logger.debug(
658 659 660
                'There are no distros available that meet %r.\n'
                'Using our best, %s.',
                str(req), best_available)
661
            return best_we_have, None
662

663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
        if self._prefer_final:
            if _final_version(best_available.parsed_version):
                if _final_version(best_we_have.parsed_version):
                    if (best_we_have.parsed_version
                        <
                        best_available.parsed_version
                        ):
                        return None, best_available
                else:
                    return None, best_available
            else:
                if (not _final_version(best_we_have.parsed_version)
                    and
                    (best_we_have.parsed_version
                     <
                     best_available.parsed_version
                     )
                    ):
                    return None, best_available
        else:
            if (best_we_have.parsed_version
                <
                best_available.parsed_version
                ):
                return None, best_available
688

689 690 691 692
        logger.debug(
            'We have the best distribution that satisfies %r.',
            str(req))
        return best_we_have, None
693

694 695 696 697 698 699 700
    def _load_dist(self, dist):
        dists = pkg_resources.Environment(
            dist.location,
            python=_get_version(self._executable),
            )[dist.project_name]
        assert len(dists) == 1
        return dists[0]
701

702
    def _call_easy_install(self, spec, ws, dest, dist):
703

704 705
        tmp = tempfile.mkdtemp(dir=dest)
        try:
reinout's avatar
reinout committed
706
            path = setuptools_loc
707 708 709
            extra_path = os.environ.get('PYTHONEXTRAPATH')
            if extra_path:
                path = os.pathsep.join([path, extra_path])
710

711 712 713
            args = ('-c', self._easy_install_cmd, '-mUNxd', _safe_arg(tmp))
            if not self._has_broken_dash_S:
                args = ('-S',) + args
714 715 716 717 718 719 720 721
            if self._always_unzip:
                args += ('-Z', )
            level = logger.getEffectiveLevel()
            if level > 0:
                args += ('-q', )
            elif level < 0:
                args += ('-v', )

sidnei's avatar
sidnei committed
722
            args += (_safe_arg(spec), )
723 724 725 726 727

            if level <= logging.DEBUG:
                logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
                             self._executable, '" "'.join(args), path)

georgyberdyshev's avatar
georgyberdyshev committed
728 729 730 731 732
            if is_jython:
                extra_env = dict(os.environ, PYTHONPATH=path)
            else:
                args += (dict(os.environ, PYTHONPATH=path), )

733
            sys.stdout.flush() # We want any pending output first
734

georgyberdyshev's avatar
georgyberdyshev committed
735 736
            if is_jython:
                exit_code = subprocess.Popen(
737
                [_safe_arg(self._executable)] + list(args),
georgyberdyshev's avatar
georgyberdyshev committed
738 739 740 741 742
                env=extra_env).wait()
            else:
                exit_code = os.spawnle(
                    os.P_WAIT, self._executable, _safe_arg (self._executable),
                    *args)
743 744 745 746 747 748 749 750

            dists = []
            env = pkg_resources.Environment(
                [tmp],
                python=_get_version(self._executable),
                )
            for project in env:
                dists.extend(env[project])
751

752 753
            if exit_code:
                logger.error(
mgedmin's avatar
mgedmin committed
754
                    "An error occurred when trying to install %s. "
755
                    "Look above this message for any errors that "
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
                    "were output by easy_install.",
                    dist)

            if not dists:
                raise zc.buildout.UserError("Couldn't install: %s" % dist)

            if len(dists) > 1:
                logger.warn("Installing %s\n"
                            "caused multiple distributions to be installed:\n"
                            "%s\n",
                            dist, '\n'.join(map(str, dists)))
            else:
                d = dists[0]
                if d.project_name != dist.project_name:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different project name.",
                                dist, d)
                if d.version != dist.version:
                    logger.warn("Installing %s\n"
                                "Caused installation of a distribution:\n"
                                "%s\n"
                                "with a different version.",
                                dist, d)

            result = []
            for d in dists:
                newloc = os.path.join(dest, os.path.basename(d.location))
                if os.path.exists(newloc):
jim's avatar
jim committed
786
                    if os.path.isdir(newloc):
787 788 789 790 791 792 793 794 795
                        shutil.rmtree(newloc)
                    else:
                        os.remove(newloc)
                os.rename(d.location, newloc)

                [d] = pkg_resources.Environment(
                    [newloc],
                    python=_get_version(self._executable),
                    )[d.project_name]
796

797 798 799
                result.append(d)

            return result
800

801 802
        finally:
            shutil.rmtree(tmp)
803

804
    def _obtain(self, requirement, source=None):
805
        # initialize out index for this project:
806
        index = self._index
807

808
        if index.obtain(requirement) is None:
809
            # Nothing is available.
810
            return None
811

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
        # Filter the available dists for the requirement and source flag.  If
        # we are not supposed to include site-packages for the given egg, we
        # also filter those out. Even if include_site_packages is False and so
        # we have excluded site packages from the _env's paths (see
        # Installer.__init__), we need to do the filtering here because an
        # .egg-link, such as one for setuptools or zc.buildout installed by
        # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
        # path in our _site_packages.
        dists = [dist for dist in index[requirement.project_name] if (
                    dist in requirement and (
                        dist.location not in self._site_packages or
                        self.allow_site_package_egg(dist.project_name))
                    and (
                        (not source) or
                        (dist.precedence == pkg_resources.SOURCE_DIST))
                    )
828 829 830 831 832 833 834 835 836 837 838 839 840
                 ]

        # If we prefer final dists, filter for final and use the
        # result if it is non empty.
        if self._prefer_final:
            fdists = [dist for dist in dists
                      if _final_version(dist.parsed_version)
                      ]
            if fdists:
                # There are final dists, so only use those
                dists = fdists

        # Now find the best one:
841 842
        best = []
        bestv = ()
843
        for dist in dists:
844 845 846 847 848 849 850 851 852
            distv = dist.parsed_version
            if distv > bestv:
                best = [dist]
                bestv = distv
            elif distv == bestv:
                best.append(dist)

        if not best:
            return None
853

854 855
        if len(best) == 1:
            return best[0]
856

857 858
        if self._download_cache:
            for dist in best:
859 860
                if (realpath(os.path.dirname(dist.location))
                    ==
861
                    self._download_cache
862
                    ):
863
                    return dist
864

865 866
        best.sort()
        return best[-1]
867

jim's avatar
jim committed
868 869
    def _fetch(self, dist, tmp, download_cache):
        if (download_cache
870
            and (realpath(os.path.dirname(dist.location)) == download_cache)
jim's avatar
jim committed
871 872
            ):
            return dist
873 874 875

        filename = get_filename_from_url(dist.location)
        new_location = os.path.join(tmp, filename)
876 877 878 879 880 881 882 883 884 885
        # Try to download from shacache first. If not possible, downloads from
        # Original location and uploads it to shacache.
        downloaded_from_cache = False
        if globals().get('_download_dir_url') \
           and globals().get('_download_dir_url') \
           and globals().get('_signature_certificate_list'):
            downloaded_from_cache = download_network_cached(_download_dir_url,
               _download_cache_url, new_location, dist.location, logger,
               _signature_certificate_list)
        if not downloaded_from_cache:
886
            new_location = self._index.download(dist.location, tmp)
887 888 889 890 891 892 893 894
            # XXX Better check for unbound global variables
            if globals().get('_upload_cache_url') \
               and globals().get('_upload_dir_url'):
                upload_network_cached(_upload_dir_url,
                   _upload_cache_url, dist.location, new_location, logger,
                   _signature_private_key_file,
                   _shacache_cert_file, _shacache_key_file,
                   _shadir_cert_file, _shadir_key_file)
jim's avatar
jim committed
895 896

        if (download_cache
897
            and (realpath(new_location) == realpath(dist.location))
jim's avatar
jim committed
898 899 900 901 902 903
            and os.path.isfile(new_location)
            ):
            # setuptools avoids making extra copies, but we want to copy
            # to the download cache
            shutil.copy2(new_location, tmp)
            new_location = os.path.join(tmp, os.path.basename(new_location))
904

jim's avatar
jim committed
905
        return dist.clone(location=new_location)
906 907 908

    def _get_dist(self, requirement, ws, always_unzip):

909
        __doing__ = 'Getting distribution for %r.', str(requirement)
910

911 912
        # Maybe an existing dist is already the best dist that satisfies the
        # requirement
913
        dist, avail = self._satisfied(requirement)
914

915 916
        if dist is None:
            if self._dest is not None:
917
                logger.info(*__doing__)
918

919 920
            # Retrieve the dist:
            if avail is None:
921
                raise MissingDistribution(requirement, ws)
922

923 924 925
            # We may overwrite distributions, so clear importer
            # cache.
            sys.path_importer_cache.clear()
926

927
            tmp = self._download_cache
jim's avatar
jim committed
928 929 930
            if tmp is None:
                tmp = tempfile.mkdtemp('get_dist')

931
            try:
jim's avatar
jim committed
932
                dist = self._fetch(avail, tmp, self._download_cache)
933

934 935
                if dist is None:
                    raise zc.buildout.UserError(
jim's avatar
jim committed
936
                        "Couldn't download distribution %s." % avail)
937

938 939
                if dist.precedence == pkg_resources.EGG_DIST:
                    # It's already an egg, just fetch it into the dest
940

941 942
                    newloc = os.path.join(
                        self._dest, os.path.basename(dist.location))
943

944 945 946 947 948
                    if os.path.isdir(dist.location):
                        # we got a directory. It must have been
                        # obtained locally.  Just copy it.
                        shutil.copytree(dist.location, newloc)
                    else:
949

950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
                        if self._always_unzip:
                            should_unzip = True
                        else:
                            metadata = pkg_resources.EggMetadata(
                                zipimport.zipimporter(dist.location)
                                )
                            should_unzip = (
                                metadata.has_metadata('not-zip-safe')
                                or
                                not metadata.has_metadata('zip-safe')
                                )

                        if should_unzip:
                            setuptools.archive_util.unpack_archive(
                                dist.location, newloc)
                        else:
                            shutil.copyfile(dist.location, newloc)

968 969
                    redo_pyc(newloc)

970 971 972 973 974 975 976 977 978 979 980 981
                    # Getting the dist from the environment causes the
                    # distribution meta data to be read.  Cloning isn't
                    # good enough.
                    dists = pkg_resources.Environment(
                        [newloc],
                        python=_get_version(self._executable),
                        )[dist.project_name]
                else:
                    # It's some other kind of dist.  We'll let easy_install
                    # deal with it:
                    dists = self._call_easy_install(
                        dist.location, ws, self._dest, dist)
982 983
                    for dist in dists:
                        redo_pyc(dist.location)
984

985 986 987
            finally:
                if tmp != self._download_cache:
                    shutil.rmtree(tmp)
988

989 990
            self._env.scan([self._dest])
            dist = self._env.best_match(requirement, ws)
991
            logger.info("Got %s.", dist)
992

993 994
        else:
            dists = [dist]
995

996
        for dist in dists:
997 998
            if (dist.has_metadata('dependency_links.txt')
                and not self._install_from_cache
999
                and self._use_dependency_links
1000
                ):
1001 1002 1003 1004 1005 1006
                for link in dist.get_metadata_lines('dependency_links.txt'):
                    link = link.strip()
                    if link not in self._links:
                        logger.debug('Adding find link %r from %s', link, dist)
                        self._links.append(link)
                        self._index = _get_index(self._executable,
1007
                                                 self._index_url, self._links,
1008
                                                 self._allow_hosts, self._path,)
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018

        for dist in dists:
            # Check whether we picked a version and, if we did, report it:
            if not (
                dist.precedence == pkg_resources.DEVELOP_DIST
                or
                (len(requirement.specs) == 1
                 and
                 requirement.specs[0][0] == '==')
                ):
1019 1020
                logger.debug('Picked: %s = %s',
                             dist.project_name, dist.version)
jim's avatar
jim committed
1021 1022 1023 1024
                if not self._allow_picked_versions:
                    raise zc.buildout.UserError(
                        'Picked: %s = %s' % (dist.project_name, dist.version)
                        )
1025 1026

        return dists
1027 1028 1029 1030

    def _maybe_add_setuptools(self, ws, dist):
        if dist.has_metadata('namespace_packages.txt'):
            for r in dist.requires():
1031
                if r.project_name in ('setuptools', 'distribute'):
1032 1033 1034 1035 1036
                    break
            else:
                # We have a namespace package but no requirement for setuptools
                if dist.precedence == pkg_resources.DEVELOP_DIST:
                    logger.warn(
1037
                        "Develop distribution: %s\n"
1038 1039 1040
                        "uses namespace packages but the distribution "
                        "does not require setuptools.",
                        dist)
jim's avatar
jim committed
1041 1042 1043
                requirement = self._constrain(
                    pkg_resources.Requirement.parse('setuptools')
                    )
1044
                if ws.find(requirement) is None:
1045 1046
                    for dist in self._get_dist(requirement, ws, False):
                        ws.add(dist)
1047 1048


jim's avatar
jim committed
1049
    def _constrain(self, requirement):
1050 1051
        if is_distribute and requirement.key == 'setuptools':
            requirement = pkg_resources.Requirement.parse('distribute')
jim's avatar
jim committed
1052 1053 1054 1055
        version = self._versions.get(requirement.project_name)
        if version:
            if version not in requirement:
                logger.error("The version, %s, is not consistent with the "
1056
                             "requirement, %r.", version, str(requirement))
jim's avatar
jim committed
1057
                raise IncompatibleVersionError("Bad version", version)
1058

jim's avatar
jim committed
1059
            requirement = pkg_resources.Requirement.parse(
1060 1061 1062
                "%s[%s] ==%s" % (requirement.project_name,
                               ','.join(requirement.extras),
                               version))
jim's avatar
jim committed
1063 1064 1065

        return requirement

1066 1067
    def install(self, specs, working_set=None):

1068
        logger.debug('Installing %s.', repr(specs)[1:-1])
1069 1070

        path = self._path
jim's avatar
jim committed
1071 1072 1073
        destination = self._dest
        if destination is not None and destination not in path:
            path.insert(0, destination)
1074

jim's avatar
jim committed
1075
        requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
1076 1077
                        for spec in specs]

1078

jim's avatar
jim committed
1079

1080 1081
        if working_set is None:
            ws = pkg_resources.WorkingSet([])
jim's avatar
jim committed
1082
        else:
1083
            ws = working_set
1084

1085
        for requirement in requirements:
1086 1087 1088
            for dist in self._get_dist(requirement, ws, self._always_unzip):
                ws.add(dist)
                self._maybe_add_setuptools(ws, dist)
1089 1090

        # OK, we have the requested distributions and they're in the working
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
        # set, but they may have unmet requirements.  We'll resolve these
        # requirements. This is code modified from
        # pkg_resources.WorkingSet.resolve.  We can't reuse that code directly
        # because we have to constrain our requirements (see
        # versions_section_ignored_for_dependency_in_favor_of_site_packages in
        # zc.buildout.tests).
        requirements.reverse() # Set up the stack.
        processed = {}  # This is a set of processed requirements.
        best = {}  # This is a mapping of key -> dist.
        # Note that we don't use the existing environment, because we want
1101 1102
        # to look for new eggs unless what we have is the best that
        # matches the requirement.
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
        env = pkg_resources.Environment(ws.entries)
        while requirements:
            # Process dependencies breadth-first.
            req = self._constrain(requirements.pop(0))
            if req in processed:
                # Ignore cyclic or redundant dependencies.
                continue
            dist = best.get(req.key)
            if dist is None:
                # Find the best distribution and add it to the map.
                dist = ws.by_key.get(req.key)
                if dist is None:
                    try:
                        dist = best[req.key] = env.best_match(req, ws)
                    except pkg_resources.VersionConflict, err:
                        raise VersionConflict(err, ws)
1119 1120 1121 1122 1123 1124 1125
                    if dist is None or (
                        dist.location in self._site_packages and not
                        self.allow_site_package_egg(dist.project_name)):
                        # If we didn't find a distribution in the
                        # environment, or what we found is from site
                        # packages and not allowed to be there, try
                        # again.
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
                        if destination:
                            logger.debug('Getting required %r', str(req))
                        else:
                            logger.debug('Adding required %r', str(req))
                        _log_requirement(ws, req)
                        for dist in self._get_dist(req,
                                                   ws, self._always_unzip):
                            ws.add(dist)
                            self._maybe_add_setuptools(ws, dist)
            if dist not in req:
                # Oops, the "best" so far conflicts with a dependency.
                raise VersionConflict(
                    pkg_resources.VersionConflict(dist, req), ws)
            requirements.extend(dist.requires(req.extras)[::-1])
            processed[req] = True
            if dist.location in self._site_packages:
                logger.debug('Egg from site-packages: %s', dist)
1143
        return ws
1144

1145
    def build(self, spec, build_ext):
1146

jim's avatar
jim committed
1147
        requirement = self._constrain(pkg_resources.Requirement.parse(spec))
1148

1149
        dist, avail = self._satisfied(requirement, 1)
1150
        if dist is not None:
1151
            return [dist.location]
1152

1153 1154 1155
        # Retrieve the dist:
        if avail is None:
            raise zc.buildout.UserError(
1156 1157
                "Couldn't find a source distribution for %r."
                % str(requirement))
1158 1159

        logger.debug('Building %r', spec)
1160

1161
        tmp = self._download_cache
jim's avatar
jim committed
1162 1163 1164
        if tmp is None:
            tmp = tempfile.mkdtemp('get_dist')

1165
        try:
jim's avatar
jim committed
1166
            dist = self._fetch(avail, tmp, self._download_cache)
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187

            build_tmp = tempfile.mkdtemp('build')
            try:
                setuptools.archive_util.unpack_archive(dist.location,
                                                       build_tmp)
                if os.path.exists(os.path.join(build_tmp, 'setup.py')):
                    base = build_tmp
                else:
                    setups = glob.glob(
                        os.path.join(build_tmp, '*', 'setup.py'))
                    if not setups:
                        raise distutils.errors.DistutilsError(
                            "Couldn't find a setup script in %s"
                            % os.path.basename(dist.location)
                            )
                    if len(setups) > 1:
                        raise distutils.errors.DistutilsError(
                            "Multiple setup scripts in %s"
                            % os.path.basename(dist.location)
                            )
                    base = os.path.dirname(setups[0])
1188

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
                setup_cfg = os.path.join(base, 'setup.cfg')
                if not os.path.exists(setup_cfg):
                    f = open(setup_cfg, 'w')
                    f.close()
                setuptools.command.setopt.edit_config(
                    setup_cfg, dict(build_ext=build_ext))

                dists = self._call_easy_install(
                    base, pkg_resources.WorkingSet(),
                    self._dest, dist)

1200 1201 1202
                for dist in dists:
                    redo_pyc(dist.location)

1203 1204 1205
                return [dist.location for dist in dists]
            finally:
                shutil.rmtree(build_tmp)
1206

1207
        finally:
1208 1209 1210
            if tmp != self._download_cache:
                shutil.rmtree(tmp)

1211 1212 1213 1214 1215
def default_versions(versions=None):
    old = Installer._versions
    if versions is not None:
        Installer._versions = versions
    return old
1216

1217 1218 1219
def download_cache(path=-1):
    old = Installer._download_cache
    if path != -1:
1220
        if path:
1221
            path = realpath(path)
1222 1223 1224
        Installer._download_cache = path
    return old

1225 1226 1227 1228 1229 1230
def install_from_cache(setting=None):
    old = Installer._install_from_cache
    if setting is not None:
        Installer._install_from_cache = bool(setting)
    return old

1231 1232 1233 1234 1235 1236
def prefer_final(setting=None):
    old = Installer._prefer_final
    if setting is not None:
        Installer._prefer_final = bool(setting)
    return old

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
def include_site_packages(setting=None):
    old = Installer._include_site_packages
    if setting is not None:
        Installer._include_site_packages = bool(setting)
    return old

def allowed_eggs_from_site_packages(setting=None):
    old = Installer._allowed_eggs_from_site_packages
    if setting is not None:
        Installer._allowed_eggs_from_site_packages = tuple(setting)
    return old

1249 1250 1251 1252 1253 1254
def use_dependency_links(setting=None):
    old = Installer._use_dependency_links
    if setting is not None:
        Installer._use_dependency_links = bool(setting)
    return old

jim's avatar
jim committed
1255 1256 1257 1258 1259 1260
def allow_picked_versions(setting=None):
    old = Installer._allow_picked_versions
    if setting is not None:
        Installer._allow_picked_versions = bool(setting)
    return old

1261 1262 1263 1264 1265 1266
def always_unzip(setting=None):
    old = Installer._always_unzip
    if setting is not None:
        Installer._always_unzip = bool(setting)
    return old

1267 1268
def install(specs, dest,
            links=(), index=None,
1269
            executable=sys.executable, always_unzip=None,
1270
            path=None, working_set=None, newest=True, versions=None,
1271
            use_dependency_links=None, allow_hosts=('*',),
1272
            include_site_packages=None, allowed_eggs_from_site_packages=None,
1273
            prefer_final=None, download_dir_url=None, download_cache_url=None,
1274
            upload_dir_url=None, upload_cache_url=None,
1275 1276 1277
            signature_certificate_list=None, signature_private_key_file=None,
            shacache_cert_file=None, shacache_key_file=None,
            shadir_cert_file=None, shadir_key_file=None):
1278 1279 1280 1281 1282
    installer = Installer(
        dest, links, index, executable, always_unzip, path, newest,
        versions, use_dependency_links, allow_hosts=allow_hosts,
        include_site_packages=include_site_packages,
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
Łukasz Nowak's avatar
Łukasz Nowak committed
1283 1284
        prefer_final=prefer_final, download_dir_url=download_dir_url,
        download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
1285
        upload_cache_url=upload_cache_url,
1286
        signature_certificate_list=signature_certificate_list,
1287 1288 1289 1290 1291
        signature_private_key_file=signature_private_key_file,
        shacache_cert_file=shacache_cert_file,
        shacache_key_file=shacache_key_file,
        shadir_cert_file=shadir_cert_file,
        shadir_key_file=shadir_key_file)
1292 1293 1294 1295 1296 1297
    return installer.install(specs, working_set)


def build(spec, dest, build_ext,
          links=(), index=None,
          executable=sys.executable,
1298 1299
          path=None, newest=True, versions=None, allow_hosts=('*',),
          include_site_packages=None, allowed_eggs_from_site_packages=None):
1300 1301 1302 1303 1304
    installer = Installer(
        dest, links, index, executable, True, path, newest, versions,
        allow_hosts=allow_hosts,
        include_site_packages=include_site_packages,
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)
1305 1306
    return installer.build(spec, build_ext)

1307

1308

1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
def _rm(*paths):
    for path in paths:
        if os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.exists(path):
            os.remove(path)

def _copyeggs(src, dest, suffix, undo):
    result = []
    undo.append(lambda : _rm(*result))
    for name in os.listdir(src):
        if name.endswith(suffix):
            new = os.path.join(dest, name)
            _rm(new)
            os.rename(os.path.join(src, name), new)
            result.append(new)
1325

1326
    assert len(result) == 1, str(result)
1327
    undo.pop()
1328

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
    return result[0]

def develop(setup, dest,
            build_ext=None,
            executable=sys.executable):

    if os.path.isdir(setup):
        directory = setup
        setup = os.path.join(directory, 'setup.py')
    else:
        directory = os.path.dirname(setup)
1340

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
    undo = []
    try:
        if build_ext:
            setup_cfg = os.path.join(directory, 'setup.cfg')
            if os.path.exists(setup_cfg):
                os.rename(setup_cfg, setup_cfg+'-develop-aside')
                def restore_old_setup():
                    if os.path.exists(setup_cfg):
                        os.remove(setup_cfg)
                    os.rename(setup_cfg+'-develop-aside', setup_cfg)
                undo.append(restore_old_setup)
            else:
                open(setup_cfg, 'w')
                undo.append(lambda: os.remove(setup_cfg))
            setuptools.command.setopt.edit_config(
                setup_cfg, dict(build_ext=build_ext))

        fd, tsetup = tempfile.mkstemp()
        undo.append(lambda: os.remove(tsetup))
        undo.append(lambda: os.close(fd))

        os.write(fd, runsetup_template % dict(
            setuptools=setuptools_loc,
            setupdir=directory,
            setup=setup,
            __file__ = setup,
            ))

        tmp3 = tempfile.mkdtemp('build', dir=dest)
1370
        undo.append(lambda : shutil.rmtree(tmp3))
1371 1372 1373 1374 1375 1376 1377 1378

        args = [
            zc.buildout.easy_install._safe_arg(tsetup),
            '-q', 'develop', '-mxN',
            '-d', _safe_arg(tmp3),
            ]

        log_level = logger.getEffectiveLevel()
1379 1380
        if log_level <= 0:
            if log_level == 0:
1381 1382 1383
                del args[1]
            else:
                args[1] == '-v'
rossp's avatar
rossp committed
1384
        if log_level < logging.DEBUG:
1385
            logger.debug("in: %r\n%s", directory, ' '.join(args))
alex_plugaru's avatar
alex_plugaru committed
1386 1387 1388 1389 1390 1391
        
        try:
            subprocess.check_call([_safe_arg(executable)] + args)
        except subprocess.CalledProcessError:
            raise zc.buildout.UserError("Installing develop egg failed")
            
1392 1393 1394 1395 1396
        return _copyeggs(tmp3, dest, '.egg-link', undo)

    finally:
        undo.reverse()
        [f() for f in undo]
1397

1398
def working_set(specs, executable, path, include_site_packages=None,
1399
                allowed_eggs_from_site_packages=None, prefer_final=None):
1400 1401 1402
    return install(
        specs, None, executable=executable, path=path,
        include_site_packages=include_site_packages,
1403 1404
        allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
        prefer_final=prefer_final)
1405

1406 1407
############################################################################
# Script generation functions
1408

1409 1410 1411 1412 1413 1414 1415 1416 1417
def scripts(
    reqs, working_set, executable, dest,
    scripts=None,
    extra_paths=(),
    arguments='',
    interpreter=None,
    initialization='',
    relative_paths=False,
    ):
1418
    """Generate scripts and/or an interpreter.
1419

1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
    See sitepackage_safe_scripts for a version that can be used with a Python
    that has code installed in site-packages. It has more options and a
    different approach.
    """
    path = _get_path(working_set, extra_paths)
    if initialization:
        initialization = '\n'+initialization+'\n'
    generated = _generate_scripts(
        reqs, working_set, dest, path, scripts, relative_paths,
        initialization, executable, arguments)
    if interpreter:
        sname = os.path.join(dest, interpreter)
        spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
        generated.extend(
            _pyscript(spath, sname, executable, rpsetup))
    return generated
1436

1437 1438 1439 1440 1441
# We need to give an alternate name to the ``scripts`` function so that it
# can be referenced within sitepackage_safe_scripts, which uses ``scripts``
# as an argument name.
_original_scripts_function = scripts

1442 1443
def sitepackage_safe_scripts(
    dest, working_set, executable, site_py_dest,
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
    reqs=(),
    scripts=None,
    interpreter=None,
    extra_paths=(),
    initialization='',
    include_site_packages=False,
    exec_sitecustomize=False,
    relative_paths=False,
    script_arguments='',
    script_initialization='',
    ):
1455 1456 1457 1458 1459 1460 1461
    """Generate scripts and/or an interpreter from a system Python.

    This accomplishes the same job as the ``scripts`` function, above,
    but it does so in an alternative way that allows safely including
    Python site packages, if desired, and  choosing to execute the Python's
    sitecustomize.
    """
1462 1463 1464 1465 1466 1467
    if _has_broken_dash_S(executable):
        if not include_site_packages:
            warnings.warn(BROKEN_DASH_S_WARNING)
        return _original_scripts_function(
            reqs, working_set, executable, dest, scripts, extra_paths,
            script_arguments, interpreter, initialization, relative_paths)
1468
    generated = []
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
    generated.append(_generate_sitecustomize(
        site_py_dest, executable, initialization, exec_sitecustomize))
    generated.append(_generate_site(
        site_py_dest, working_set, executable, extra_paths,
        include_site_packages, relative_paths))
    script_initialization = _script_initialization_template % dict(
        site_py_dest=site_py_dest,
        script_initialization=script_initialization)
    if not script_initialization.endswith('\n'):
        script_initialization += '\n'
    generated.extend(_generate_scripts(
        reqs, working_set, dest, [site_py_dest], scripts, relative_paths,
        script_initialization