Add support for installing from .win32.exe's created by distutils (by

converting them to eggs).  Bump version to 0.5a1.

--HG--
branch : setuptools
extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041070

EasyInstall.txt

@@ -23,14 +23,14 @@ Installing "Easy Install"
 -------------------------
 
 Windows users can just download and run the `setuptools binary installer for
-Windows <http://peak.telecommunity.com/dist/setuptools-0.4a4.win32.exe>`_.
+Windows <http://peak.telecommunity.com/dist/setuptools-0.5a1.win32.exe>`_.
 All others should just download `ez_setup.py
 <http://peak.telecommunity.com/dist/ez_setup.py>`_, and run it; this will
 download and install the correct version of ``setuptools`` for your Python
 version.  You may receive a message telling you about an obsolete version of
 setuptools being present; if so, you must be sure to delete it entirely, along
 with the old ``pkg_resources`` module if it's present on ``sys.path``.
-    
+
 An ``easy_install.py`` script will be installed in the normal location for
 Python scripts on your platform. In the examples below, you'll need to replace
 references to ``easy_install`` with the correct invocation to run
@@ -62,7 +62,7 @@ version, and automatically downloading, building, and installing it::
 **Example 2**. Install or upgrade a package by name and version by finding
 links on a given "download page"::
 
-    easy_install -f http://peak.telecommunity.com/dist "setuptools>=0.4a4"
+    easy_install -f http://peak.telecommunity.com/dist "setuptools>=0.5a1"
 
 **Example 3**. Download a source distribution from a specified URL,
 automatically building and installing it::
@@ -78,9 +78,9 @@ Easy Install accepts URLs, filenames, PyPI package names (i.e., ``distutils``
 attempt to locate the latest available version that meets your criteria.
 
 When downloading or processing downloaded files, Easy Install recognizes
-distutils *source* (not binary) distribution files with extensions of .tgz,
-.tar, .tar.gz, .tar.bz2, or .zip.  And of course it handles already-built .egg
-distributions as well.
+distutils source distribution files with extensions of .tgz, .tar, .tar.gz,
+.tar.bz2, or .zip.  And of course it handles already-built .egg
+distributions as well as ``.win32.exe`` installers built using distutils.
 
 By default, packages are installed to the running Python installation's
 ``site-packages`` directory, unless you provide the ``-d`` or ``--install-dir``
@@ -268,7 +268,7 @@ and Windows, respectively), and finally a ``distutils.cfg`` file in the
 
     # set the default location to install packages
     install_dir = /home/me/lib/python
-    
+
     # Notice that indentation can be used to continue an option
     # value; this is especially useful for the "--find-links"
     # option, which tells easy_install to use download links on
@@ -442,6 +442,24 @@ Known Issues
  * There's no automatic retry for borked Sourceforge mirrors, which can easily
    time out or be missing a file.
 
+0.5a1
+ * Added support for converting ``.win32.exe`` installers to eggs on the fly.
+   EasyInstall will now recognize such files by name and install them.
+
+ * Added support for "self-installation" bootstrapping.  Packages can now
+   include ``ez_setup.py`` in their source distribution, and add the following
+   to their ``setup.py``, in order to automatically bootstrap installation of
+   setuptools as part of their setup process::
+
+    from ez_setup import use_setuptools
+    use_setuptools()
+
+    from setuptools import setup
+    # etc...
+
+ * Fixed a problem with picking the "best" version to install (versions were
+   being sorted as strings, rather than as parsed values)
+
 0.4a4
  * Added support for the distutils "verbose/quiet" and "dry-run" options, as
    well as the "optimize" flag.
@@ -465,7 +483,7 @@ Known Issues
 
 0.4a2
  * Added support for installing scripts
- 
+
  * Added support for setting options via distutils configuration files, and
    using distutils' default options as a basis for EasyInstall's defaults.
 
@@ -558,9 +576,6 @@ Future Plans
 ============
 
 * Process the installed package's dependencies as well as the base package
-* Support "self-installation" - bootstrapping setuptools install into another
-  package's installation process (copy egg, write setuptools.pth)
-* Support installation from bdist_wininst packages?
 * Additional utilities to list/remove/verify packages
 * Signature checking?  SSL?  Ability to suppress PyPI search?
 * Display byte progress meter when downloading distributions and long pages?

easy_install.py

@@ -12,7 +12,7 @@ __ http://peak.telecommunity.com/DevCenter/EasyInstall
 
 """
 
-import sys, os.path, zipimport, shutil, tempfile
+import sys, os.path, zipimport, shutil, tempfile, zipfile
 
 from setuptools import Command
 from setuptools.sandbox import run_setup
@@ -20,9 +20,14 @@ from distutils import log, dir_util
 from distutils.sysconfig import get_python_lib
 from distutils.errors import DistutilsArgError, DistutilsOptionError
 from setuptools.archive_util import unpack_archive
-from setuptools.package_index import PackageIndex
+from setuptools.package_index import PackageIndex, parse_bdist_wininst
+from setuptools.command import bdist_egg
 from pkg_resources import *
 
+__all__ = [
+    'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg',
+    'main', 'get_exe_prefixes',
+]
 
 def samefile(p1,p2):
     if hasattr(os.path,'samefile') and (
@@ -34,11 +39,6 @@ def samefile(p1,p2):
         os.path.normpath(os.path.normcase(p2))
     )
 
-
-
-
-
-
 class easy_install(Command):
     """Manage a download/build/install process"""
 
@@ -249,6 +249,9 @@ class easy_install(Command):
         if dist_filename.lower().endswith('.egg'):
             return [self.install_egg(dist_filename, True, tmpdir)]
 
+        if dist_filename.lower().endswith('.exe'):
+            return [self.install_exe(dist_filename, tmpdir)]
+
         # Anything else, try to extract and build
         if os.path.isfile(dist_filename):
             unpack_archive(dist_filename, tmpdir, self.unpack_progress)
@@ -282,9 +285,6 @@ class easy_install(Command):
 
 
 
-
-
-
     def install_egg(self, egg_path, zip_ok, tmpdir):
         destination = os.path.join(self.install_dir,os.path.basename(egg_path))
         destination = os.path.abspath(destination)
@@ -326,6 +326,88 @@ class easy_install(Command):
         self.update_pth(dist)
         return dist
 
+    def install_exe(self, dist_filename, tmpdir):
+        # See if it's valid, get data
+        cfg = extract_wininst_cfg(dist_filename)
+        if cfg is None:
+            raise RuntimeError(
+                "%s is not a valid distutils Windows .exe" % dist_filename
+            )
+
+        # Create a dummy distribution object until we build the real distro
+        dist = Distribution(None,
+            name=cfg.get('metadata','name'),
+            version=cfg.get('metadata','version'),
+            platform="win32"
+        )
+
+        # Convert the .exe to an unpacked egg
+        egg_path = dist.path = os.path.join(tmpdir, dist.egg_name()+'.egg')
+        egg_tmp  = egg_path+'.tmp'
+        self.exe_to_egg(dist_filename, egg_tmp)
+
+        # Write EGG-INFO/PKG-INFO
+        pkg_inf = os.path.join(egg_tmp, 'EGG-INFO', 'PKG-INFO')
+        f = open(pkg_inf,'w')
+        f.write('Metadata-Version: 1.0\n')
+        for k,v in cfg.items('metadata'):
+            if k<>'target_version':
+                f.write('%s: %s\n' % (k.replace('_','-').title(), v))
+        f.close()
+
+        # Build .egg file from tmpdir
+        bdist_egg.make_zipfile(
+            egg_path, egg_tmp,
+            verbose=self.verbose, dry_run=self.dry_run
+        )
+
+        # install the .egg        
+        return self.install_egg(egg_path, self.zip_ok, tmpdir)
+
+
+
+
+    def exe_to_egg(self, dist_filename, egg_tmp):
+        """Extract a bdist_wininst to the directories an egg would use"""
+
+        # Check for .pth file and set up prefix translations
+        prefixes = get_exe_prefixes(dist_filename)
+        to_compile = []
+        native_libs = []
+
+        def process(src,dst):
+            for old,new in prefixes:
+                if src.startswith(old):
+                    src = new+src[len(old):]
+                    dst = os.path.join(egg_tmp, *src.split('/'))
+                    dl = dst.lower()
+                    if dl.endswith('.pyd') or dl.endswith('.dll'):
+                        native_libs.append(src)
+                    elif dl.endswith('.py') and old!='SCRIPTS/':
+                        to_compile.append(dst)
+                    return dst
+            if not src.endswith('.pth'):
+                log.warn("WARNING: can't process %s", src)
+            return None
+
+        # extract, tracking .pyd/.dll->native_libs and .py -> to_compile
+        unpack_archive(dist_filename, egg_tmp, process)
+       
+        for res in native_libs:
+            if res.lower().endswith('.pyd'):    # create stubs for .pyd's
+                parts = res.split('/')
+                resource, parts[-1] = parts[-1], parts[-1][:-1]
+                pyfile = os.path.join(egg_tmp, *parts)
+                to_compile.append(pyfile)
+                bdist_egg.write_stub(resource, pyfile)
+        
+        self.byte_compile(to_compile)   # compile .py's
+        
+        if native_libs:     # write EGG-INFO/native_libs.txt
+            nl_txt = os.path.join(egg_tmp, 'EGG-INFO', 'native_libs.txt')
+            ensure_directory(nl_txt)
+            open(nl_txt,'w').write('\n'.join(native_libs)+'\n')
+
     def installation_report(self, dist):
         """Helpful installation message for display to package users"""
 
@@ -355,20 +437,19 @@ PYTHONPATH, or by being added to sys.path by your code.)
         version = dist.version
         return msg % locals()
 
-    def update_pth(self,dist):
-        if self.pth_file is not None:
-            remove = self.pth_file.remove
-            for d in self.pth_file.get(dist.key,()):    # drop old entries
-                log.info("Removing %s from .pth file", d)
-                remove(d)
-            if not self.multi_version:
-                log.info("Adding %s to .pth file", dist)
-                self.pth_file.add(dist) # add new entry
-            self.pth_file.save()
+
+
+
+
+
+
+
+
+
+
 
 
     def build_egg(self, tmpdir, setup_script):
-        from setuptools.command import bdist_egg
         sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
 
         args = ['bdist_egg']
@@ -391,27 +472,28 @@ PYTHONPATH, or by being added to sys.path by your code.)
         finally:
             log.set_verbosity(self.verbose) # restore our log verbosity
 
+    def update_pth(self,dist):
+        if self.pth_file is not None:
+            remove = self.pth_file.remove
+            for d in self.pth_file.get(dist.key,()):    # drop old entries
+                log.info("Removing %s from .pth file", d)
+                remove(d)
+            if not self.multi_version:
+                log.info("Adding %s to .pth file", dist)
+                self.pth_file.add(dist) # add new entry
+            self.pth_file.save()
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+            if dist.name=='setuptools':
+                # Ensure that setuptools itself never becomes unavailable!
+                f = open(os.path.join(self.install_dir,'setuptools.pth'), 'w')
+                f.write(dist.path+'\n')
+                f.close()
 
 
     def unpack_progress(self, src, dst):
         # Progress filter for unpacking
         log.debug("Unpacking %s to %s", src, dst)
-        return True     # only unpack-and-compile skips files for dry run
+        return dst     # only unpack-and-compile skips files for dry run
 
 
     def unpack_and_compile(self, egg_path, destination):
@@ -421,10 +503,13 @@ PYTHONPATH, or by being added to sys.path by your code.)
             if dst.endswith('.py'):
                 to_compile.append(dst)
             self.unpack_progress(src,dst)
-            return not self.dry_run
+            return not self.dry_run and dst or None
 
         unpack_archive(egg_path, destination, pf)
+        self.byte_compile(to_compile)
+
 
+    def byte_compile(self, to_compile):
         from distutils.util import byte_compile
         try:
             # try to make the byte compile messages quieter
@@ -446,6 +531,85 @@ PYTHONPATH, or by being added to sys.path by your code.)
 
 
 
+def extract_wininst_cfg(dist_filename):
+    """Extract configuration data from a bdist_wininst .exe
+
+    Returns a ConfigParser.RawConfigParser, or None
+    """
+    f = open(dist_filename,'rb')
+    try:
+        endrec = zipfile._EndRecData(f)
+        if endrec is None:
+            return None
+
+        prepended = (endrec[9] - endrec[5]) - endrec[6]
+        if prepended < 12:  # no wininst data here
+            return None               
+        f.seek(prepended-12)
+
+        import struct, StringIO, ConfigParser
+        tag, cfglen, bmlen = struct.unpack("<iii",f.read(12))
+        if tag<>0x1234567A:
+            return None     # not a valid tag
+
+        f.seek(prepended-(12+cfglen+bmlen))
+        cfg = ConfigParser.RawConfigParser({'version':'','target_version':''})
+        try:
+            cfg.readfp(StringIO.StringIO(f.read(cfglen)))
+        except ConfigParser.Error:
+            return None
+        if not cfg.has_section('metadata') or not cfg.has_section('Setup'):
+            return None
+        return cfg              
+
+    finally:
+        f.close()
+
+
+
+
+
+
+
+
+def get_exe_prefixes(exe_filename):
+    """Get exe->egg path translations for a given .exe file"""
+
+    prefixes = [
+        ('PURELIB/', ''),
+        ('PLATLIB/', ''),
+        ('SCRIPTS/', 'EGG-INFO/scripts/')
+    ]
+    z = zipfile.ZipFile(exe_filename)
+    try:
+        for info in z.infolist():
+            name = info.filename
+            if not name.endswith('.pth'):
+                continue
+            parts = name.split('/')
+            if len(parts)<>2:
+                continue
+            if parts[0] in ('PURELIB','PLATLIB'):
+                pth = z.read(name).strip()
+                prefixes[0] = ('PURELIB/%s/' % pth), ''
+                prefixes[1] = ('PLATLIB/%s/' % pth), ''
+                break
+    finally:
+        z.close()
+
+    return prefixes
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

ez_setup.py

@@ -14,7 +14,7 @@ the appropriate options to ``use_setuptools()``.
 This file can also be run as a script to install or upgrade setuptools.
 """
 
-DEFAULT_VERSION = "0.4a4"
+DEFAULT_VERSION = "0.5a1"
 DEFAULT_URL     = "http://peak.telecommunity.com/dist/"
 
 import sys, os

setup.py

 #!/usr/bin/env python
 """Distutils setup file, used to install or test 'setuptools'"""
 
-VERSION = "0.4a4"
+VERSION = "0.5a1"
 from setuptools import setup, find_packages, Require
 
 setup(

setuptools/__init__.py

@@ -8,7 +8,7 @@ from distutils.core import Command as _Command
 from distutils.util import convert_path
 import os.path
 
-__version__ = '0.4a4'
+__version__ = '0.5a1'
 
 __all__ = [
     'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require',

setuptools/package_index.py

@@ -9,13 +9,25 @@ URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
 EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
 
 __all__ = [
-    'PackageIndex', 'distros_for_url', 
+    'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
+    'interpret_distro_name',
 ]
 
 
+def parse_bdist_wininst(name):
+    """Return (base,pyversion) or (None,None) for possible .exe name"""
 
+    lower = name.lower()
+    base, py_ver = None, None
 
+    if lower.endswith('.exe'):
+        if lower.endswith('.win32.exe'):
+            base = name[:-10]
+        elif lower[-16:].startswith('.win32-py'):
+            py_ver = base[-7:-4]
+            base = name[:-16]
 
+    return base,py_ver
 
 
 
@@ -27,8 +39,32 @@ __all__ = [
 
 
 
+def distros_for_url(url, metadata=None):
+    """Yield egg or source distribution objects that might be found at a URL"""
 
+    path = urlparse.urlparse(url)[2]
+    base = urllib2.unquote(path.split('/')[-1])
+    
+    if base.endswith('.egg'):
+        dist = Distribution.from_filename(base, metadata)
+        dist.path = url
+        return [dist]   # only one, unambiguous interpretation
 
+    if base.endswith('.exe'):
+        win_base, py_ver = parse_bdist_wininst(name)
+        if win_base is not None:
+            return interpret_distro_name(
+                url, win_base, metadata, py_ver, BINARY_DIST, "win32"
+            )
+
+    # Try source distro extensions (.zip, .tgz, etc.)
+    #            
+    for ext in EXTENSIONS:
+        if base.endswith(ext):
+            base = base[:-len(ext)]               
+            return interpret_distro_name(url, base, metadata)
+
+    return []  # no extension matched
 
 
 
@@ -39,24 +75,14 @@ __all__ = [
 
 
 
-def distros_for_url(url, metadata=None):
-    """Yield egg or source distribution objects that might be found at a URL"""
 
-    path = urlparse.urlparse(url)[2]
-    base = urllib2.unquote(path.split('/')[-1])
 
-    if base.endswith('.egg'):
-        dist = Distribution.from_filename(base, metadata)
-        dist.path = url
-        yield dist
-        return  # only one, unambiguous interpretation
 
-    for ext in EXTENSIONS:
-        if base.endswith(ext):
-            base = base[:-len(ext)]
-            break
-    else:
-        return  # no extension matched
+
+    
+def interpret_distro_name(url, base, metadata,
+    py_version=None, distro_type=SOURCE_DIST, platform=None
+):
 
     # Generate alternative interpretations of a source distro name
     # Because some packages are ambiguous as to name/versions split
@@ -74,12 +100,27 @@ def distros_for_url(url, metadata=None):
     for p in range(1,len(parts)+1):
         yield Distribution(
             url, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
-            distro_type = SOURCE_DIST
+            py_version=py_version, distro_type = distro_type,
+            platform = platform
         )
 
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 class PackageIndex(AvailableDistributions):
     """A distribution index that scans web pages for download URLs"""