Skip to content

S
setuptools
Commit 43e70738 authored by PJ Eby's avatar PJ Eby
Browse files
Options

Split setup-running and archive-extraction utilities into separate modules, 

for easy use by tools other than EasyInstall.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041049
parent 5bb11a1a
Hide whitespace changes
Inline Side-by-side
Showing with 402 additions and 275 deletions
EasyInstall.txt
View file @ 43e70738
......@@ -270,6 +270,10 @@ Command-Line Options
Release Notes/Change History
============================
Known Issues
* There's no automatic retry for borked Sourceforge mirrors, which can easily
time out or be missing a file.
0.4a1
* Added ``--scan-url`` and ``--index-url`` options, to scan download pages
and search PyPI for needed packages.
......
easy_install.py
View file @ 43e70738
......@@ -17,19 +17,17 @@ import os.path
import pkg_resources
import re
import zipimport
import zipfile
import tarfile
import shutil
import urlparse
import urllib
import tempfile
import __builtin__
from setuptools.sandbox import run_setup
from setuptools.archive_util import unpack_archive
from distutils.sysconfig import get_python_lib
from shutil import rmtree # must have, because it can be called from __del__
from pkg_resources import *
_os = sys.modules[os.name]
_open = open
class Opener(urllib.FancyURLopener):
def http_error_default(self, url, fp, errcode, errmsg, headers):
......@@ -39,6 +37,8 @@ class Opener(urllib.FancyURLopener):
return info
opener = Opener()
HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
......@@ -251,7 +251,7 @@ class Installer:
# Anything else, try to extract and build
if os.path.isfile(dist_filename):
self._extract_file(dist_filename)
unpack_archive(dist_filename, self.tmpdir) # XXX add progress log
# Find the setup.py file
from glob import glob
......@@ -268,7 +268,14 @@ class Installer:
)
setup_script = setups[0]
self._run_setup(setup_script)
from setuptools.command import bdist_egg
sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
try:
run_setup(setup_script, '-q', 'bdist_egg')
except SystemExit, v:
raise RuntimeError(
"Setup script exited with %s" % (v.args[0],)
)
eggs = []
for egg in glob(
......@@ -278,95 +285,6 @@ class Installer:
return eggs
def _extract_zip(self,zipname,extract_dir):
z = zipfile.ZipFile(zipname)
try:
for info in z.infolist():
name = info.filename
# don't extract absolute paths or ones with .. in them
if name.startswith('/') or '..' in name:
continue
target = os.path.join(extract_dir,name)
if name.endswith('/'):
# directory
ensure_directory(target)
else:
# file
ensure_directory(target)
data = z.read(info.filename)
f = open(target,'wb')
try:
f.write(data)
finally:
f.close()
del data
finally:
z.close()
def _extract_tar(self,tarobj):
try:
tarobj.chown = lambda *args: None # don't do any chowning!
for member in tarobj:
if member.isfile() or member.isdir():
name = member.name
# don't extract absolute paths or ones with .. in them
if not name.startswith('/') and '..' not in name:
tarobj.extract(member,self.tmpdir)
finally:
tarobj.close()
def _run_setup(self, setup_script):
from setuptools.command import bdist_egg
sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
old_dir = os.getcwd()
save_argv = sys.argv[:]
save_path = sys.path[:]
try:
os.chdir(os.path.dirname(setup_script))
try:
sys.argv[:] = [setup_script, '-q', 'bdist_egg']
sys.path.insert(0,os.getcwd())
DirectorySandbox(self.tmpdir).run(
lambda: execfile(
"setup.py",
{'__file__':setup_script, '__name__':'__main__'}
)
)
except SystemExit, v:
if v.args and v.args[0]:
raise RuntimeError(
"Setup script exited with %s" % (v.args[0],)
)
finally:
os.chdir(old_dir)
sys.path[:] = save_path
sys.argv[:] = save_argv
def install_egg(self, egg_path, zip_ok):
destination = os.path.join(self.instdir, os.path.basename(egg_path))
......@@ -389,7 +307,7 @@ class Installer:
else:
os.mkdir(destination)
self._extract_zip(egg_path, destination)
unpack_archive(egg_path, destination) # XXX add progress??
if os.path.isdir(destination):
dist = Distribution.from_filename(
......@@ -449,20 +367,6 @@ class Installer:
def _extract_file(self, dist_filename):
if zipfile.is_zipfile(dist_filename):
self._extract_zip(dist_filename, self.tmpdir)
else:
try:
tar = tarfile.open(dist_filename)
except tarfile.TarError:
raise RuntimeError(
"Not a valid tar or zip archive: %s" % dist_filename
)
else:
self._extract_tar(tar)
def _download_html(self, url, headers, filename):
# Check for a sourceforge URL
sf_url = url.startswith('http://prdownloads.')
......@@ -494,6 +398,16 @@ class Installer:
os.system("svn checkout -q %s %s" % (url, filename))
return filename
def _download_sourceforge(self, source_url, sf_page):
"""Download package from randomly-selected SourceForge mirror"""
......@@ -531,6 +445,10 @@ class Installer:
def installation_report(self, dist):
"""Helpful installation message for display to package users"""
......@@ -572,170 +490,6 @@ PYTHONPATH, or by being added to sys.path by your code.)
class AbstractSandbox:
"""Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
_active = False
def __init__(self):
self._attrs = [
name for name in dir(_os)
if not name.startswith('_') and hasattr(self,name)
]
def _copy(self, source):
for name in self._attrs:
setattr(os, name, getattr(source,name))
def run(self, func):
"""Run 'func' under os sandboxing"""
try:
self._copy(self)
__builtin__.open = __builtin__.file = self._open
self._active = True
return func()
finally:
self._active = False
__builtin__.open = __builtin__.file = _open
self._copy(_os)
def _mk_dual_path_wrapper(name):
original = getattr(_os,name)
def wrap(self,src,dst,*args,**kw):
if self._active:
src,dst = self._remap_pair(name,src,dst,*args,**kw)
return original(src,dst,*args,**kw)
return wrap
for name in ["rename", "link", "symlink"]:
if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)
def _mk_single_path_wrapper(name, original=None):
original = original or getattr(_os,name)
def wrap(self,path,*args,**kw):
if self._active:
path = self._remap_input(name,path,*args,**kw)
return original(path,*args,**kw)
return wrap
_open = _mk_single_path_wrapper('file', _open)
for name in [
"stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
"remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
"startfile", "mkfifo", "mknod", "pathconf", "access"
]:
if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)
def _mk_single_with_return(name):
original = getattr(_os,name)
def wrap(self,path,*args,**kw):
if self._active:
path = self._remap_input(name,path,*args,**kw)
return self._remap_output(name, original(path,*args,**kw))
return original(path,*args,**kw)
return wrap
for name in ['readlink', 'tempnam']:
if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)
def _mk_query(name):
original = getattr(_os,name)
def wrap(self,*args,**kw):
retval = original(*args,**kw)
if self._active:
return self._remap_output(name, retval)
return retval
return wrap
for name in ['getcwd', 'tmpnam']:
if hasattr(_os,name): locals()[name] = _mk_query(name)
def _validate_path(self,path):
"""Called to remap or validate any path, whether input or output"""
return path
def _remap_input(self,operation,path,*args,**kw):
"""Called for path inputs"""
return self._validate_path(path)
def _remap_output(self,operation,path):
"""Called for path outputs"""
return self._validate_path(path)
def _remap_pair(self,operation,src,dst,*args,**kw):
"""Called for path pairs like rename, link, and symlink operations"""
return (
self._remap_input(operation+'-from',src,*args,**kw),
self._remap_input(operation+'-to',dst,*args,**kw)
)
class DirectorySandbox(AbstractSandbox):
"""Restrict operations to a single subdirectory - pseudo-chroot"""
write_ops = dict.fromkeys([
"open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
"utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
])
def __init__(self,sandbox):
self._sandbox = os.path.realpath(sandbox)
self._prefix = os.path.join(self._sandbox,'')
AbstractSandbox.__init__(self)
def _violation(self, operation, *args, **kw):
raise SandboxViolation(operation, args, kw)
def _open(self, path, mode='r', *args, **kw):
if mode not in ('r', 'rt', 'rb', 'rU') and not self._ok(path):
self._violation("open", path, mode, *args, **kw)
return _open(path,mode,*args,**kw)
def tmpnam(self):
self._violation("tmpnam")
def _ok(self,path):
active = self._active
try:
self._active = False
realpath = os.path.realpath(path)
if realpath==self._sandbox or realpath.startswith(self._prefix):
return True
finally:
self._active = active
def _remap_input(self,operation,path,*args,**kw):
"""Called for path inputs"""
if operation in self.write_ops and not self._ok(path):
self._violation(operation, os.path.realpath(path), *args, **kw)
return path
def _remap_pair(self,operation,src,dst,*args,**kw):
"""Called for path pairs like rename, link, and symlink operations"""
if not self._ok(src) or not self._ok(dst):
self._violation(operation, src, dst, *args, **kw)
return (src,dst)
class SandboxViolation(RuntimeError):
"""A setup script attempted to modify the filesystem outside the sandbox"""
def __str__(self):
return """SandboxViolation: %s%r %s
The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.
This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand. Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args
class PthDistributions(AvailableDistributions):
"""A .pth file with Distribution paths in it"""
......
setuptools/archive_util.py 0 → 100755
View file @ 43e70738
"""Utilities for extracting common archive formats"""
__all__ = [
"unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter",
"UnrecognizedFormat", "extraction_drivers"
]
import zipfile, tarfile, os
from pkg_resources import ensure_directory
class UnrecognizedFormat(RuntimeError):
"""Couldn't recognize the archive type"""
def default_filter(src,dst):
"""The default progress/filter callback; returns True for all files"""
return True
def unpack_archive(filename, extract_dir, progress_filter=default_filter,
drivers=None
):
"""Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat``
`progress_filter` is a function taking two arguments: a source path
internal to the archive ('/'-separated), and a filesystem path where it
will be extracted. The callback must return a true value, or else that
file or directory will be skipped. The callback can thus be used to
report on the progress of the extraction, as well as to filter the items
extracted.
`drivers`, if supplied, must be a non-empty sequence of functions with the
same signature as this function (minus the `drivers` argument), that raise
``UnrecognizedFormat`` if they do not support extracting the designated
archive type. The `drivers` are tried in sequence until one is found that
does not raise an error, or until all are exhausted (in which case
``UnrecognizedFormat`` is raised). If you do not supply a sequence of
drivers, the module's ``extraction_drivers`` constant will be used, which
means that ``unpack_zipfile`` and ``unpack_tarfile`` will be tried, in that
order.
"""
for driver in drivers or extraction_drivers:
try:
driver(filename, extract_dir, progress_filter)
except UnrecognizedFormat:
continue
else:
return
else:
raise UnrecognizedFormat(
"Not a recognized archive type: %s" % filename
)
def unpack_zipfile(filename, extract_dir, progress_filter=default_filter):
"""Unpack zip `filename` to `extract_dir`
Raises ``UnrecognizedFormat`` if `filename` is not a zipfile (as determined
by ``zipfile.is_zipfile()``). See ``unpack_archive()`` for an explanation
of the `progress_filter` argument.
"""
if not zipfile.is_zipfile(filename):
raise UnrecognizedFormat("%s is not a zip file" % (filename,))
z = zipfile.ZipFile(filename)
try:
for info in z.infolist():
name = info.filename
# don't extract absolute paths or ones with .. in them
if name.startswith('/') or '..' in name:
continue
target = os.path.join(extract_dir,name)
if not progress_filter(name,target):
continue
if name.endswith('/'):
# directory
ensure_directory(target)
else:
# file
ensure_directory(target)
data = z.read(info.filename)
f = open(target,'wb')
try:
f.write(data)
finally:
f.close()
del data
finally:
z.close()
def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):
"""Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`
Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined
by ``tarfile.open()``). See ``unpack_archive()`` for an explanation
of the `progress_filter` argument.
"""
try:
tarobj = tarfile.open(filename)
except tarfile.TarError:
raise UnrecognizedFormat(
"%s is not a compressed or uncompressed tar file" % (filename,)
)
try:
tarobj.chown = lambda *args: None # don't do any chowning!
for member in tarobj:
if member.isfile() or member.isdir():
name = member.name
# don't extract absolute paths or ones with .. in them
if not name.startswith('/') and '..' not in name:
dst = os.path.join(extract_dir, *name.split('/'))
if progress_filter(name, dst):
tarobj.extract(member,extract_dir)
return True
finally:
tarobj.close()
extraction_drivers = unpack_zipfile, unpack_tarfile
setuptools/sandbox.py 0 → 100755
View file @ 43e70738
import os, sys, __builtin__
_os = sys.modules[os.name]
_open = open
__all__ = [
"AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]
def run_setup(setup_script, *args):
"""Run a distutils setup script, sandboxed in its directory"""
old_dir = os.getcwd()
save_argv = sys.argv[:]
save_path = sys.path[:]
setup_dir = os.path.abspath(os.path.dirname(setup_script))
try:
os.chdir(setup_dir)
try:
sys.argv[:] = [setup_script]+list(args)
sys.path.insert(0, setup_dir)
DirectorySandbox(setup_dir).run(
lambda: execfile(
"setup.py",
{'__file__':setup_script, '__name__':'__main__'}
)
)
except SystemExit, v:
if v.args and v.args[0]:
raise
# Normal exit, just return
finally:
os.chdir(old_dir)
sys.path[:] = save_path
sys.argv[:] = save_argv
class AbstractSandbox:
"""Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
_active = False
def __init__(self):
self._attrs = [
name for name in dir(_os)
if not name.startswith('_') and hasattr(self,name)
]
def _copy(self, source):
for name in self._attrs:
setattr(os, name, getattr(source,name))
def run(self, func):
"""Run 'func' under os sandboxing"""
try:
self._copy(self)
__builtin__.open = __builtin__.file = self._open
self._active = True
return func()
finally:
self._active = False
__builtin__.open = __builtin__.file = _open
self._copy(_os)
def _mk_dual_path_wrapper(name):
original = getattr(_os,name)
def wrap(self,src,dst,*args,**kw):
if self._active:
src,dst = self._remap_pair(name,src,dst,*args,**kw)
return original(src,dst,*args,**kw)
return wrap
for name in ["rename", "link", "symlink"]:
if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)
def _mk_single_path_wrapper(name, original=None):
original = original or getattr(_os,name)
def wrap(self,path,*args,**kw):
if self._active:
path = self._remap_input(name,path,*args,**kw)
return original(path,*args,**kw)
return wrap
_open = _mk_single_path_wrapper('file', _open)
for name in [
"stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
"remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
"startfile", "mkfifo", "mknod", "pathconf", "access"
]:
if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)
def _mk_single_with_return(name):
original = getattr(_os,name)
def wrap(self,path,*args,**kw):
if self._active:
path = self._remap_input(name,path,*args,**kw)
return self._remap_output(name, original(path,*args,**kw))
return original(path,*args,**kw)
return wrap
for name in ['readlink', 'tempnam']:
if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)
def _mk_query(name):
original = getattr(_os,name)
def wrap(self,*args,**kw):
retval = original(*args,**kw)
if self._active:
return self._remap_output(name, retval)
return retval
return wrap
for name in ['getcwd', 'tmpnam']:
if hasattr(_os,name): locals()[name] = _mk_query(name)
def _validate_path(self,path):
"""Called to remap or validate any path, whether input or output"""
return path
def _remap_input(self,operation,path,*args,**kw):
"""Called for path inputs"""
return self._validate_path(path)
def _remap_output(self,operation,path):
"""Called for path outputs"""
return self._validate_path(path)
def _remap_pair(self,operation,src,dst,*args,**kw):
"""Called for path pairs like rename, link, and symlink operations"""
return (
self._remap_input(operation+'-from',src,*args,**kw),
self._remap_input(operation+'-to',dst,*args,**kw)
)
class DirectorySandbox(AbstractSandbox):
"""Restrict operations to a single subdirectory - pseudo-chroot"""
write_ops = dict.fromkeys([
"open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
"utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
])
def __init__(self,sandbox):
self._sandbox = os.path.realpath(sandbox)
self._prefix = os.path.join(self._sandbox,'')
AbstractSandbox.__init__(self)
def _violation(self, operation, *args, **kw):
raise SandboxViolation(operation, args, kw)
def _open(self, path, mode='r', *args, **kw):
if mode not in ('r', 'rt', 'rb', 'rU') and not self._ok(path):
self._violation("open", path, mode, *args, **kw)
return _open(path,mode,*args,**kw)
def tmpnam(self):
self._violation("tmpnam")
def _ok(self,path):
active = self._active
try:
self._active = False
realpath = os.path.realpath(path)
if realpath==self._sandbox or realpath.startswith(self._prefix):
return True
finally:
self._active = active
def _remap_input(self,operation,path,*args,**kw):
"""Called for path inputs"""
if operation in self.write_ops and not self._ok(path):
self._violation(operation, os.path.realpath(path), *args, **kw)
return path
def _remap_pair(self,operation,src,dst,*args,**kw):
"""Called for path pairs like rename, link, and symlink operations"""
if not self._ok(src) or not self._ok(dst):
self._violation(operation, src, dst, *args, **kw)
return (src,dst)
class SandboxViolation(RuntimeError):
"""A setup script attempted to modify the filesystem outside the sandbox"""
def __str__(self):
return """SandboxViolation: %s%r %s
The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.
This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand. Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7