Skip to content

S
setuptools
Commit 449f3574 authored by PJ Eby's avatar PJ Eby
Browse files
Options

Rebalance responsibilities between PackageIndex, Installer, and main() so 

that PackageIndex handles all downloading of any kind, Installers can be
reused for multiple packages, and main() manages temporary directories and
all communication between PackageIndex and Installer.  Also, change
run_setup to take an argument sequence, because later we will probably need
other arguments to control other aspects of run_setup's behavior.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041050
parent 43e70738
Show whitespace changes
Inline Side-by-side
Showing with 264 additions and 206 deletions
EasyInstall.txt
View file @ 449f3574
...@@ -274,6 +274,23 @@ Known Issues ...@@ -274,6 +274,23 @@ Known Issues
* There's no automatic retry for borked Sourceforge mirrors, which can easily * There's no automatic retry for borked Sourceforge mirrors, which can easily
time out or be missing a file. time out or be missing a file.
0.4a2
* Use ``urllib2`` instead of ``urllib``, to allow use of ``https:`` URLs if
Python includes SSL support.
* All downloads are now managed by the ``PackageIndex`` class (which is now
subclassable and replaceable), so that embedders can more easily override
download logic, give download progress reports, etc.
* The ``Installer`` class no longer handles downloading, manages a temporary
directory, or tracks the ``zip_ok`` option. Downloading is now handled
by ``PackageIndex``, and the latter two are now managed by ``main()``.
* There is a new ``setuptools.sandbox.run_setup()`` API to invoke a setup
script in a directory sandbox, and a new ``setuptools.archive_util`` module
with an ``unpack_archive()`` API. These were split out of EasyInstall to
allow reuse by other tools and applications.
0.4a1 0.4a1
* Added ``--scan-url`` and ``--index-url`` options, to scan download pages * Added ``--scan-url`` and ``--index-url`` options, to scan download pages
and search PyPI for needed packages. and search PyPI for needed packages.
......
easy_install.py
View file @ 449f3574
...@@ -19,23 +19,23 @@ import re ...@@ -19,23 +19,23 @@ import re
import zipimport import zipimport
import shutil import shutil
import urlparse import urlparse
import urllib import urllib2
import tempfile import tempfile
from setuptools.sandbox import run_setup from setuptools.sandbox import run_setup
from setuptools.archive_util import unpack_archive from setuptools.archive_util import unpack_archive
from distutils.sysconfig import get_python_lib from distutils.sysconfig import get_python_lib
from shutil import rmtree # must have, because it can be called from __del__
from pkg_resources import * from pkg_resources import *
class Opener(urllib.FancyURLopener):
def http_error_default(self, url, fp, errcode, errmsg, headers):
"""Default error handling -- don't raise an exception."""
info = urllib.addinfourl(fp, headers, "http:" + url)
info.status, info.reason = errcode, errmsg
return info
opener = Opener()
...@@ -46,7 +46,7 @@ def distros_for_url(url, metadata=None): ...@@ -46,7 +46,7 @@ def distros_for_url(url, metadata=None):
"""Yield egg or source distribution objects that might be found at a URL""" """Yield egg or source distribution objects that might be found at a URL"""
path = urlparse.urlparse(url)[2] path = urlparse.urlparse(url)[2]
base = urllib.unquote(path.split('/')[-1]) base = urllib2.unquote(path.split('/')[-1])
if base.endswith('.egg'): if base.endswith('.egg'):
dist = Distribution.from_filename(base, metadata) dist = Distribution.from_filename(base, metadata)
...@@ -71,7 +71,7 @@ def distros_for_url(url, metadata=None): ...@@ -71,7 +71,7 @@ def distros_for_url(url, metadata=None):
# compare lower than any numeric version number, and is therefore unlikely # compare lower than any numeric version number, and is therefore unlikely
# to match a request for it. It's still a potential problem, though, and # to match a request for it. It's still a potential problem, though, and
# in the long run PyPI and the distutils should go for "safe" names and # in the long run PyPI and the distutils should go for "safe" names and
# versions in source distribution names. # versions in distribution archive names (sdist and bdist).
parts = base.split('-') parts = base.split('-')
for p in range(1,len(parts)+1): for p in range(1,len(parts)+1):
...@@ -105,7 +105,7 @@ class PackageIndex(AvailableDistributions): ...@@ -105,7 +105,7 @@ class PackageIndex(AvailableDistributions):
# don't need the actual page # don't need the actual page
return return
f = opener.open(url) f = self.open_url(url)
self.fetched_urls[url] = self.fetched_urls[f.url] = True self.fetched_urls[url] = self.fetched_urls[f.url] = True
if 'html' not in f.headers['content-type'].lower(): if 'html' not in f.headers['content-type'].lower():
f.close() # not html, we can't process it f.close() # not html, we can't process it
...@@ -134,7 +134,7 @@ class PackageIndex(AvailableDistributions): ...@@ -134,7 +134,7 @@ class PackageIndex(AvailableDistributions):
def scan(link): def scan(link):
if link.startswith(self.index_url): if link.startswith(self.index_url):
parts = map( parts = map(
urllib.unquote, link[len(self.index_url):].split('/') urllib2.unquote, link[len(self.index_url):].split('/')
) )
if len(parts)==2: if len(parts)==2:
# it's a package page, sanitize and index it # it's a package page, sanitize and index it
...@@ -162,69 +162,25 @@ class PackageIndex(AvailableDistributions): ...@@ -162,69 +162,25 @@ class PackageIndex(AvailableDistributions):
if dist in requirement: if dist in requirement:
return dist return dist
class Installer: def download(self, spec, tmpdir):
"""Manage a download/build/install process""" """Locate and/or download `spec`, returning a local filename
pth_file = None `spec` may be a ``Requirement`` object, or a string containing a URL,
cleanup = False an existing local filename, or a package/version requirement spec
(i.e. the string form of a ``Requirement`` object).
def __init__(self, If necessary, the requirement is searched for in the package index.
instdir=None, zip_ok=False, multi=None, tmpdir=None, index=None If the download is successful, the return value is a local file path,
): and it is a subpath of `tmpdir` if the distribution had to be
if index is None: downloaded. If no matching distribution is found, return ``None``.
index = AvailableDistributions() Various errors may be raised if a problem occurs during downloading.
if tmpdir is None: """
tmpdir = tempfile.mkdtemp(prefix="easy_install-")
self.cleanup = True
elif not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
self.tmpdir = os.path.realpath(tmpdir)
site_packages = get_python_lib() if not isinstance(spec,Requirement):
if instdir is None or self.samefile(site_packages,instdir):
instdir = site_packages
self.pth_file = PthDistributions(
os.path.join(instdir,'easy-install.pth')
)
elif multi is None:
multi = True
elif not multi:
# explicit false, raise an error
raise RuntimeError(
"Can't do single-version installs outside site-packages"
)
self.index = index
self.instdir = instdir
self.zip_ok = zip_ok
self.multi = multi
def close(self):
if self.cleanup and os.path.isdir(self.tmpdir):
rmtree(self.tmpdir,True)
def __del__(self):
self.close()
def samefile(self,p1,p2):
if hasattr(os.path,'samefile') and (
os.path.exists(p1) and os.path.exists(p2)
):
return os.path.samefile(p1,p2)
return (
os.path.normpath(os.path.normcase(p1)) ==
os.path.normpath(os.path.normcase(p2))
)
def download(self, spec):
"""Locate and/or download or `spec`, returning a local filename"""
if isinstance(spec,Requirement):
pass
else:
scheme = URL_SCHEME(spec) scheme = URL_SCHEME(spec)
if scheme: if scheme:
# It's a url, download it to self.tmpdir # It's a url, download it to tmpdir
return self._download_url(scheme.group(1), spec) return self._download_url(scheme.group(1), spec, tmpdir)
elif os.path.exists(spec): elif os.path.exists(spec):
# Existing file or directory, just return it # Existing file or directory, just return it
...@@ -239,126 +195,87 @@ class Installer: ...@@ -239,126 +195,87 @@ class Installer:
) )
# process a Requirement # process a Requirement
dist = self.index.best_match(spec,[]) dist = self.best_match(spec,[])
if dist is not None: if dist is not None:
return self.download(dist.path) return self.download(dist.path, tmpdir)
return None return None
def install_eggs(self, dist_filename):
# .egg dirs or files are already built, so just return them
if dist_filename.lower().endswith('.egg'):
return [self.install_egg(dist_filename,True)]
# Anything else, try to extract and build
if os.path.isfile(dist_filename):
unpack_archive(dist_filename, self.tmpdir) # XXX add progress log
# Find the setup.py file dl_blocksize = 8192
from glob import glob
setup_script = os.path.join(self.tmpdir, 'setup.py')
if not os.path.exists(setup_script):
setups = glob(os.path.join(self.tmpdir, '*', 'setup.py'))
if not setups:
raise RuntimeError(
"Couldn't find a setup script in %s" % dist_filename
)
if len(setups)>1:
raise RuntimeError(
"Multiple setup scripts in %s" % dist_filename
)
setup_script = setups[0]
from setuptools.command import bdist_egg def _download_to(self, url, filename):
sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg) # Download the file
fp, tfp = None, None
try: try:
run_setup(setup_script, '-q', 'bdist_egg') fp = self.open_url(url)
except SystemExit, v: if isinstance(fp, urllib2.HTTPError):
raise RuntimeError( raise RuntimeError(
"Setup script exited with %s" % (v.args[0],) "Can't download %s: %s %s" % (url, fp.code,fp.msg)
) )
eggs = [] headers = fp.info()
for egg in glob( blocknum = 0
os.path.join(os.path.dirname(setup_script),'dist','*.egg') bs = self.dl_blocksize
): size = -1
eggs.append(self.install_egg(egg, self.zip_ok))
if "content-length" in headers:
return eggs size = int(headers["Content-Length"])
self.reporthook(url, filename, blocknum, bs, size)
def install_egg(self, egg_path, zip_ok):
tfp = open(filename,'wb')
while True:
block = fp.read(bs)
if block:
tfp.write(block)
blocknum += 1
self.reporthook(url, filename, blocknum, bs, size)
else:
break
return headers
destination = os.path.join(self.instdir, os.path.basename(egg_path)) finally:
ensure_directory(destination) if fp: fp.close()
if tfp: tfp.close()
if not self.samefile(egg_path, destination): def reporthook(self, url, filename, blocknum, blksize, size):
if os.path.isdir(destination): pass # no-op
shutil.rmtree(destination)
elif os.path.isfile(destination):
os.unlink(destination)
if zip_ok:
if egg_path.startswith(self.tmpdir):
shutil.move(egg_path, destination)
else:
shutil.copy2(egg_path, destination)
elif os.path.isdir(egg_path):
shutil.move(egg_path, destination)
else: def open_url(self, url):
os.mkdir(destination) try:
unpack_archive(egg_path, destination) # XXX add progress?? return urllib2.urlopen(url)
except urllib2.HTTPError, v:
return v
except urllib2.URLError, v:
raise RuntimeError("Download error: %s" % v.reason)
if os.path.isdir(destination):
dist = Distribution.from_filename(
destination, metadata=PathMetadata(
destination, os.path.join(destination,'EGG-INFO')
)
)
else:
metadata = EggMetadata(zipimport.zipimporter(destination))
dist = Distribution.from_filename(destination,metadata=metadata)
self.index.add(dist)
if self.pth_file is not None:
map(self.pth_file.remove, self.pth_file.get(dist.key,())) # drop old
if not self.multi:
self.pth_file.add(dist) # add new
self.pth_file.save()
return dist
def _download_url(self, scheme, url): def _download_url(self, scheme, url, tmpdir):
# Determine download filename # Determine download filename
name = filter(None,urlparse.urlparse(url)[2].split('/'))[-1] #
name = filter(None,urlparse.urlparse(url)[2].split('/'))
if name:
name = name[-1]
while '..' in name: while '..' in name:
name = name.replace('..','.').replace('\\','_') name = name.replace('..','.').replace('\\','_')
else:
name = "__downloaded__" # default if URL has no path contents
filename = os.path.join(self.tmpdir,name) filename = os.path.join(tmpdir,name)
# Download the file
#
if scheme=='svn' or scheme.startswith('svn+'): if scheme=='svn' or scheme.startswith('svn+'):
return self._download_svn(url, filename) return self._download_svn(url, filename)
# Download the file
class _opener(urllib.FancyURLopener):
http_error_default = urllib.URLopener.http_error_default
try:
filename,headers = _opener().retrieve(
url,filename
)
except IOError,v:
if v.args and v.args[0]=='http error':
raise RuntimeError(
"Download error: %s %s" % v.args[1:3]
)
else: else:
raise headers = self._download_to(url, filename)
if 'html' in headers['content-type'].lower(): if 'html' in headers['content-type'].lower():
return self._download_html(url, headers, filename) return self._download_html(url, headers, filename, tmpdir)
else:
# and return its filename
return filename return filename
...@@ -367,7 +284,8 @@ class Installer: ...@@ -367,7 +284,8 @@ class Installer:
def _download_html(self, url, headers, filename):
def _download_html(self, url, headers, filename, tmpdir):
# Check for a sourceforge URL # Check for a sourceforge URL
sf_url = url.startswith('http://prdownloads.') sf_url = url.startswith('http://prdownloads.')
file = open(filename) file = open(filename)
...@@ -388,7 +306,7 @@ class Installer: ...@@ -388,7 +306,7 @@ class Installer:
page = file.read() page = file.read()
file.close() file.close()
os.unlink(filename) os.unlink(filename)
return self._download_sourceforge(url, page) return self._download_sourceforge(url, page, tmpdir)
break # not an index page break # not an index page
file.close() file.close()
raise RuntimeError("Unexpected HTML page found at "+url) raise RuntimeError("Unexpected HTML page found at "+url)
...@@ -408,7 +326,7 @@ class Installer: ...@@ -408,7 +326,7 @@ class Installer:
def _download_sourceforge(self, source_url, sf_page): def _download_sourceforge(self, source_url, sf_page, tmpdir):
"""Download package from randomly-selected SourceForge mirror""" """Download package from randomly-selected SourceForge mirror"""
mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)') mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)')
...@@ -420,7 +338,7 @@ class Installer: ...@@ -420,7 +338,7 @@ class Installer:
import random import random
url = urlparse.urljoin(source_url, random.choice(urls)) url = urlparse.urljoin(source_url, random.choice(urls))
f = urllib.urlopen(url) f = self.open_url(url)
match = re.search( match = re.search(
r'<META HTTP-EQUIV="refresh" content=".*?URL=(.*?)"', r'<META HTTP-EQUIV="refresh" content=".*?URL=(.*?)"',
f.read() f.read()
...@@ -430,7 +348,7 @@ class Installer: ...@@ -430,7 +348,7 @@ class Installer:
if match: if match:
download_url = match.group(1) download_url = match.group(1)
scheme = URL_SCHEME(download_url) scheme = URL_SCHEME(download_url)
return self._download_url(scheme.group(1), download_url) return self._download_url(scheme.group(1), download_url, tmpdir)
else: else:
raise RuntimeError( raise RuntimeError(
'No META HTTP-EQUIV="refresh" found in Sourceforge page at %s' 'No META HTTP-EQUIV="refresh" found in Sourceforge page at %s'
...@@ -449,6 +367,129 @@ class Installer: ...@@ -449,6 +367,129 @@ class Installer:
class Installer:
"""Manage a download/build/install process"""
pth_file = None
cleanup = False
def __init__(self, instdir=None, multi=None):
site_packages = get_python_lib()
if instdir is None or self.samefile(site_packages,instdir):
instdir = site_packages
self.pth_file = PthDistributions(
os.path.join(instdir,'easy-install.pth')
)
elif multi is None:
multi = True
elif not multi:
# explicit false, raise an error
raise RuntimeError(
"Can't do single-version installs outside site-packages"
)
self.instdir = instdir
self.multi = multi
def samefile(self,p1,p2):
if hasattr(os.path,'samefile') and (
os.path.exists(p1) and os.path.exists(p2)
):
return os.path.samefile(p1,p2)
return (
os.path.normpath(os.path.normcase(p1)) ==
os.path.normpath(os.path.normcase(p2))
)
def install_eggs(self, dist_filename, zip_ok, tmpdir):
# .egg dirs or files are already built, so just return them
if dist_filename.lower().endswith('.egg'):
return [self.install_egg(dist_filename, True, tmpdir)]
# Anything else, try to extract and build
if os.path.isfile(dist_filename):
unpack_archive(dist_filename, tmpdir) # XXX add progress log
# Find the setup.py file
from glob import glob
setup_script = os.path.join(tmpdir, 'setup.py')
if not os.path.exists(setup_script):
setups = glob(os.path.join(tmpdir, '*', 'setup.py'))
if not setups:
raise RuntimeError(
"Couldn't find a setup script in %s" % dist_filename
)
if len(setups)>1:
raise RuntimeError(
"Multiple setup scripts in %s" % dist_filename
)
setup_script = setups[0]
from setuptools.command import bdist_egg
sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
try:
run_setup(setup_script, ['-q', 'bdist_egg'])
except SystemExit, v:
raise RuntimeError(
"Setup script exited with %s" % (v.args[0],)
)
eggs = []
for egg in glob(
os.path.join(os.path.dirname(setup_script),'dist','*.egg')
):
eggs.append(self.install_egg(egg, zip_ok, tmpdir))
return eggs
def install_egg(self, egg_path, zip_ok, tmpdir):
destination = os.path.join(self.instdir, os.path.basename(egg_path))
ensure_directory(destination)
if not self.samefile(egg_path, destination):
if os.path.isdir(destination):
shutil.rmtree(destination)
elif os.path.isfile(destination):
os.unlink(destination)
if zip_ok:
if egg_path.startswith(tmpdir):
shutil.move(egg_path, destination)
else:
shutil.copy2(egg_path, destination)
elif os.path.isdir(egg_path):
shutil.move(egg_path, destination)
else:
os.mkdir(destination)
unpack_archive(egg_path, destination) # XXX add progress??
if os.path.isdir(destination):
dist = Distribution.from_filename(
destination, metadata=PathMetadata(
destination, os.path.join(destination,'EGG-INFO')
)
)
else:
metadata = EggMetadata(zipimport.zipimporter(destination))
dist = Distribution.from_filename(destination,metadata=metadata)
self.update_pth(dist)
return dist
def installation_report(self, dist): def installation_report(self, dist):
"""Helpful installation message for display to package users""" """Helpful installation message for display to package users"""
...@@ -478,14 +519,14 @@ PYTHONPATH, or by being added to sys.path by your code.) ...@@ -478,14 +519,14 @@ PYTHONPATH, or by being added to sys.path by your code.)
version = dist.version version = dist.version
return msg % locals() return msg % locals()
def update_pth(self,dist):
if self.pth_file is not None:
remove = self.pth_file.remove
for d in self.pth_file.get(dist.key,()): # drop old entries
remove(d)
if not self.multi:
self.pth_file.add(dist) # add new entry
self.pth_file.save()
...@@ -533,7 +574,7 @@ class PthDistributions(AvailableDistributions): ...@@ -533,7 +574,7 @@ class PthDistributions(AvailableDistributions):
URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
def main(argv, factory=Installer): def main(argv, installer_type=Installer, index_type=PackageIndex):
from optparse import OptionParser from optparse import OptionParser
...@@ -572,44 +613,44 @@ def main(argv, factory=Installer): ...@@ -572,44 +613,44 @@ def main(argv, factory=Installer):
def alloc_tmp():
if options.tmpdir is None:
return tempfile.mkdtemp(prefix="easy_install-")
elif not os.path.isdir(options.tmpdir):
os.makedirs(options.tmpdir)
return os.path.realpath(options.tmpdir)
try: try:
index = PackageIndex(options.index_url) index = index_type(options.index_url)
inst = installer_type(options.instdir, options.multi)
if options.scan_urls: if options.scan_urls:
for url in options.scan_urls: for url in options.scan_urls:
index.scan_url(url) index.scan_url(url)
for spec in args: for spec in args:
inst = factory( tmpdir = alloc_tmp()
options.instdir, options.zip_ok, options.multi, options.tmpdir,
index
)
try: try:
print "Downloading", spec print "Downloading", spec
downloaded = inst.download(spec) download = index.download(spec, tmpdir)
if downloaded is None: if download is None:
raise RuntimeError( raise RuntimeError(
"Could not find distribution for %r" % spec "Could not find distribution for %r" % spec
) )
print "Installing", os.path.basename(downloaded)
for dist in inst.install_eggs(downloaded): print "Installing", os.path.basename(download)
for dist in inst.install_eggs(download,options.zip_ok, tmpdir):
index.add(dist)
print inst.installation_report(dist) print inst.installation_report(dist)
finally: finally:
inst.close() if options.tmpdir is None:
shutil.rmtree(tmpdir)
except RuntimeError, v: except RuntimeError, v:
print >>sys.stderr,"error:",v print >>sys.stderr,"error:",v
sys.exit(1) sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[1:]) main(sys.argv[1:])
setuptools/sandbox.py
View file @ 449f3574
...@@ -8,7 +8,7 @@ __all__ = [ ...@@ -8,7 +8,7 @@ __all__ = [
] ]
def run_setup(setup_script, *args): def run_setup(setup_script, args):
"""Run a distutils setup script, sandboxed in its directory""" """Run a distutils setup script, sandboxed in its directory"""
old_dir = os.getcwd() old_dir = os.getcwd()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7