Merge with Distribute 0.6.39

--HG--
rename : distribute_setup.py => ez_setup.py

.hgignore

@@ -11,3 +11,4 @@ bin
 include
 \.Python
 *.swp
+CHANGES (links).txt

.hgtags

@@ -44,3 +44,8 @@ b1a7f86b315a1f8c20036d718d6dc641bb84cac6 0.6.32
 6acac3919ae9a7dba2cbecbe3d4b31ece25d5f09 0.6.33
 23c310bf4ae8e4616e37027f08891702f5a33bc9 0.6.34
 2abe1117543be0edbafb10c7c159d1bcb1cb1b87 0.6.35
+c813a29e831f266d427d4a4bce3da97f475a8eee 0.6.36
+be6f65eea9c10ce78b6698d8c220b6e5de577292 0.6.37
+2b26ec8909bff210f47c5f8fc620bc505e1610b5 0.6.37
+f0d502a83f6c83ba38ad21c15a849c2daf389ec7 0.6.38
+d737b2039c5f92af8000f78bbc80b6a5183caa97 0.6.39

CHANGES.txt

@@ -2,6 +2,45 @@
 CHANGES
 =======
 
+------
+0.6.39
+------
+
+* Add support for console launchers on ARM platforms.
+* Fix possible issue in GUI launchers where the subsystem was not supplied to
+  the linker.
+* Launcher build script now refactored for robustness.
+* Issue #375: Resources extracted from a zip egg to the file system now also
+  check the contents of the file against the zip contents during each
+  invocation of get_resource_filename.
+
+------
+0.6.38
+------
+
+* Issue #371: The launcher manifest file is now installed properly.
+
+------
+0.6.37
+------
+
+* Issue #143: Launcher scripts, including easy_install itself, are now
+  accompanied by a manifest on 32-bit Windows environments to avoid the
+  Installer Detection Technology and thus undesirable UAC elevation described
+  in `this Microsoft article
+  <http://technet.microsoft.com/en-us/library/cc709628%28WS.10%29.aspx>`_.
+
+------
+0.6.36
+------
+
+* Pull Request #35: In `Buildout issue 64
+  <https://github.com/buildout/buildout/issues/64>`_, it was reported that
+  under Python 3, installation of distutils scripts could attempt to copy
+  the ``__pycache__`` directory as a file, causing an error, apparently only
+  under Windows. Easy_install now skips all directories when processing
+  metadata scripts.
+
 ------
 0.6.35
 ------

CONTRIBUTORS.txt

@@ -8,6 +8,7 @@ Contributors
 * Christophe Combelles
 * Daniel Stutzbach
 * Daniel Holth
+* Grigory Petrov
 * Hanno Schlichting
 * Jannis Leidel
 * Jason R. Coombs

MANIFEST.in

-recursive-include setuptools *.py *.txt *.exe
+recursive-include setuptools *.py *.txt *.exe *.xml
 recursive-include tests *.py *.c *.pyx *.txt
 recursive-include setuptools/tests *.html
 recursive-include docs *.py *.txt *.conf *.css *.css_t Makefile indexsidebar.html

docs/conf.py

@@ -48,9 +48,9 @@ copyright = u'2009-2013, The fellowship of the packaging'
 # built documents.
 #
 # The short X.Y version.
-version = '0.6.36'
+version = '0.6.40'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.36'
+release = '0.6.40'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

ez_setup.py

@@ -47,7 +47,7 @@ except ImportError:
             args = [quote(arg) for arg in args]
         return os.spawnl(os.P_WAIT, sys.executable, *args) == 0
 
-DEFAULT_VERSION = "0.6.36"
+DEFAULT_VERSION = "0.6.40"
 DEFAULT_URL = "http://pypi.python.org/packages/source/s/setuptools/"
 
 

launcher.c

@@ -14,6 +14,14 @@
        gcc -DGUI=0           -mno-cygwin -O -s -o setuptools/cli.exe launcher.c
        gcc -DGUI=1 -mwindows -mno-cygwin -O -s -o setuptools/gui.exe launcher.c
 
+    To build for Windows RT, install both Visual Studio Express for Windows 8
+    and for Windows Desktop (both freeware), create "win32" application using
+    "Windows Desktop" version, create new "ARM" target via
+    "Configuration Manager" menu and modify ".vcxproj" file by adding
+    "<WindowsSDKDesktopARMSupport>true</WindowsSDKDesktopARMSupport>" tag
+    as child of "PropertyGroup" tags that has "Debug|ARM" and "Release|ARM"
+    properties.
+
     It links to msvcrt.dll, but this shouldn't be a problem since it doesn't
     actually run Python in the same process.  Note that using 'exec' instead
     of 'spawn' doesn't work, because on Windows this leads to the Python

msvc-build-launcher.cmd

 @echo off
 
-REM VCVARSALL may be in Program Files or Program Files (x86)
-PATH=C:\Program Files\Microsoft Visual Studio 9.0\VC;%PATH%
-PATH=C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC;%PATH%
+REM Use old Windows SDK 6.1 so created .exe will be compatible with
+REM old Windows versions.
+REM Windows SDK 6.1 may be downloaded at:
+REM  http://www.microsoft.com/en-us/download/details.aspx?id=11310
+set PATH_OLD=%PATH%
+
+REM The SDK creates a false install of Visual Studio at one of these locations
+set PATH=C:\Program Files\Microsoft Visual Studio 9.0\VC\bin;%PATH%
+set PATH=C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin;%PATH%
 
 REM set up the environment to compile to x86
-call VCVARSALL x86
-cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x86 /out:setuptools/cli-32.exe
-cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x86 /out:setuptools/gui-32.exe
+call VCVARS32
+if "%ERRORLEVEL%"=="0" (
+  cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x86 /SUBSYSTEM:CONSOLE /out:setuptools/cli-32.exe
+  cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x86 /SUBSYSTEM:WINDOWS /out:setuptools/gui-32.exe
+) else (
+  echo Windows SDK 6.1 not found to build Windows 32-bit version
+)
 
 REM now for 64-bit
-call VCVARSALL x86_amd64
-cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x64 /out:setuptools/cli-64.exe
-cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x64 /out:setuptools/gui-64.exe
\ No newline at end of file
+REM Use the x86_amd64 profile, which is the 32-bit cross compiler for amd64
+call VCVARSx86_amd64
+if "%ERRORLEVEL%"=="0" (
+  cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x64 /SUBSYSTEM:CONSOLE /out:setuptools/cli-64.exe
+  cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" launcher.c /O2 /link /MACHINE:x64 /SUBSYSTEM:WINDOWS /out:setuptools/gui-64.exe
+) else (
+  echo Windows SDK 6.1 not found to build Windows 64-bit version
+)
+
+REM Windows RT ARM build requires both freeware
+REM "Visual Studio Express 2012 for Windows 8" and
+REM "Visual Studio Express 2012 for Windows Desktop" to be installed from
+REM http://www.microsoft.com/visualstudio/eng/products/visual-studio-express-products
+set PATH=%PATH_OLD%
+set PATH=C:\Program Files\Microsoft Visual Studio 11.0\VC;%PATH%
+set PATH=C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC;%PATH%
+call VCVARSALL x86_arm >nul 2>&1
+if "%ERRORLEVEL%"=="0" (
+  echo Building Windows RT Version ...
+  cl /D "GUI=0" /D "WIN32_LEAN_AND_MEAN" /D _ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE launcher.c /O2 /link /MACHINE:ARM /SUBSYSTEM:CONSOLE /out:setuptools/cli-arm-32.exe
+  cl /D "GUI=1" /D "WIN32_LEAN_AND_MEAN" /D _ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE launcher.c /O2 /link /MACHINE:ARM /SUBSYSTEM:WINDOWS /out:setuptools/gui-arm-32.exe
+) else (
+  echo Visual Studio ^(Express^) 2012 not found to build Windows RT Version
+)
+
+set PATH=%PATH_OLD%
+

pkg_resources.py

@@ -1384,6 +1384,16 @@ class ZipProvider(EggProvider):
                 self._extract_resource(manager, self._eager_to_zip(name))
         return self._extract_resource(manager, zip_path)
 
+    @staticmethod
+    def _get_date_and_size(zip_stat):
+        t,d,size = zip_stat[5], zip_stat[6], zip_stat[3]
+        date_time = (
+            (d>>9)+1980, (d>>5)&0xF, d&0x1F,                      # ymd
+            (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1   # hms, etc.
+        )
+        timestamp = time.mktime(date_time)
+        return timestamp, size
+
     def _extract_resource(self, manager, zip_path):
 
         if zip_path in self._index():
@@ -1393,27 +1403,18 @@ class ZipProvider(EggProvider):
                 )
             return os.path.dirname(last)  # return the extracted directory name
 
-        zip_stat = self.zipinfo[zip_path]
-        t,d,size = zip_stat[5], zip_stat[6], zip_stat[3]
-        date_time = (
-            (d>>9)+1980, (d>>5)&0xF, d&0x1F,                      # ymd
-            (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1   # hms, etc.
-        )
-        timestamp = time.mktime(date_time)
+        timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
 
-        try:
         if not WRITE_SUPPORT:
             raise IOError('"os.rename" and "os.unlink" are not supported '
                           'on this platform')
+        try:
 
             real_path = manager.get_cache_path(
                 self.egg_name, self._parts(zip_path)
             )
 
-            if os.path.isfile(real_path):
-                stat = os.stat(real_path)
-                if stat.st_size==size and stat.st_mtime==timestamp:
-                    # size and stamp match, don't bother extracting
+            if self._is_current(real_path, zip_path):
                 return real_path
 
             outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path))
@@ -1427,11 +1428,9 @@ class ZipProvider(EggProvider):
 
             except os.error:
                 if os.path.isfile(real_path):
-                    stat = os.stat(real_path)
-
-                    if stat.st_size==size and stat.st_mtime==timestamp:
-                        # size and stamp match, somebody did it just ahead of
-                        # us, so we're done
+                    if self._is_current(real_path, zip_path):
+                        # the file became current since it was checked above,
+                        #  so proceed.
                         return real_path
                     elif os.name=='nt':     # Windows, del old file and retry
                         unlink(real_path)
@@ -1444,6 +1443,23 @@ class ZipProvider(EggProvider):
 
         return real_path
 
+    def _is_current(self, file_path, zip_path):
+        """
+        Return True if the file_path is current for this zip_path
+        """
+        timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
+        if not os.path.isfile(file_path):
+            return False
+        stat = os.stat(file_path)
+        if stat.st_size!=size or stat.st_mtime!=timestamp:
+            return False
+        # check that the contents match
+        zip_contents = self.loader.get_data(zip_path)
+        f = open(file_path, 'rb')
+        file_contents = f.read()
+        f.close()
+        return zip_contents == file_contents
+
     def _get_eager_resources(self):
         if self.eagers is None:
             eagers = []

release.py

@@ -14,13 +14,17 @@ import sys
 import urllib2
 import getpass
 import collections
+import itertools
+import re
 
 try:
 	import keyring
 except Exception:
 	pass
 
-VERSION = '0.6.36'
+VERSION = '0.6.40'
+PACKAGE_INDEX = 'https://pypi.python.org/pypi'
+PACKAGE_INDEX = 'https://pypi.python.org/pypi'
 
 def get_next_version():
 	digits = map(int, VERSION.split('.'))
@@ -50,7 +54,7 @@ def get_mercurial_creds(system='https://bitbucket.org', username=None):
 	# todo: consider getting this from .hgrc
 	username = username or getpass.getuser()
 	keyring_username = '@@'.join((username, system))
-	system = '@'.join((keyring_username, 'Mercurial'))
+	system = 'Mercurial'
 	password = (
 		keyring.get_password(system, keyring_username)
 		if 'keyring' in globals()
@@ -110,13 +114,20 @@ def do_release():
 
 	subprocess.check_call(['hg', 'update', VERSION])
 
+	linkify('CHANGES.txt', 'CHANGES (links).txt')
+
 	has_docs = build_docs()
 	if os.path.isdir('./dist'):
 		shutil.rmtree('./dist')
-	cmd = [sys.executable, 'setup.py', '-q', 'egg_info', '-RD', '-b', '',
-		'sdist', 'register', 'upload']
+	cmd = [
+		sys.executable, 'setup.py', '-q',
+		'egg_info', '-RD', '-b', '',
+		'sdist',
+		'register', '-r', PACKAGE_INDEX,
+		'upload', '-r', PACKAGE_INDEX,
+	]
 	if has_docs:
-		cmd.append('upload_docs')
+		cmd.extend(['upload_docs', '-r', PACKAGE_INDEX])
 	subprocess.check_call(cmd)
 	upload_bootstrap_script()
 
@@ -166,5 +177,59 @@ def upload_bootstrap_script():
 	except:
 		print("Unable to upload bootstrap script. Ask Tarek to do it.")
 
+def linkify(source, dest):
+	with open(source) as source:
+		out = _linkified_text(source.read())
+	with open(dest, 'w') as dest:
+		dest.write(out)
+
+def _linkified(rst_path):
+	"return contents of reStructureText file with linked issue references"
+	rst_file = open(rst_path)
+	rst_content = rst_file.read()
+	rst_file.close()
+
+	return _linkified_text(rst_content)
+
+def _linkified_text(rst_content):
+	# first identify any existing HREFs so they're not changed
+	HREF_pattern = re.compile('`.*?`_', re.MULTILINE | re.DOTALL)
+
+	# split on the HREF pattern, returning the parts to be linkified
+	plain_text_parts = HREF_pattern.split(rst_content)
+	anchors = []
+	linkified_parts = [_linkified_part(part, anchors)
+		for part in plain_text_parts]
+	pairs = itertools.izip_longest(
+		linkified_parts,
+		HREF_pattern.findall(rst_content),
+		fillvalue='',
+	)
+	rst_content = ''.join(flatten(pairs))
+
+	anchors = sorted(anchors)
+
+	bitroot = 'http://bitbucket.org/tarek/distribute'
+	rst_content += "\n"
+	for x in anchors:
+		issue = re.findall(r'\d+', x)[0]
+		rst_content += '.. _`%s`: %s/issue/%s\n' % (x, bitroot, issue)
+	rst_content += "\n"
+	return rst_content
+
+def flatten(listOfLists):
+	"Flatten one level of nesting"
+	return itertools.chain.from_iterable(listOfLists)
+
+
+def _linkified_part(text, anchors):
+	"""
+	Linkify a part and collect any anchors generated
+	"""
+	revision = re.compile(r'\b(issue\s+#?\d+)\b', re.M | re.I)
+
+	anchors.extend(revision.findall(text)) # ['Issue #43', ...]
+	return revision.sub(r'`\1`_', text)
+
 if __name__ == '__main__':
 	do_release()

setup.py

@@ -46,7 +46,7 @@ exec(init_file.read(), d)
 init_file.close()
 
 SETUP_COMMANDS = d['__all__']
-VERSION = "0.6.36"
+VERSION = "0.6.40"
 
 from setuptools import setup, find_packages
 from setuptools.command.build_py import build_py as _build_py
@@ -107,28 +107,16 @@ class test(_test):
                 f.close()
 
 
-# return contents of reStructureText file with linked issue references
-def _linkified(rst_path):
-    bitroot = 'http://bitbucket.org/tarek/distribute'
-    revision = re.compile(r'\b(issue\s+#?\d+)\b', re.M | re.I)
-
-    rst_file = open(rst_path)
-    rst_content = rst_file.read()
-    rst_file.close()
-
-    anchors = revision.findall(rst_content) # ['Issue #43', ...]
-    anchors = sorted(set(anchors))
-    rst_content = revision.sub(r'`\1`_', rst_content)
-    rst_content += "\n"
-    for x in anchors:
-        issue = re.findall(r'\d+', x)[0]
-        rst_content += '.. _`%s`: %s/issue/%s\n' % (x, bitroot, issue)
-    rst_content += "\n"
-    return rst_content
-
 readme_file = open('README.txt')
-long_description = readme_file.read() + _linkified('CHANGES.txt')
+# the release script adds hyperlinks to issues
+if os.path.exists('CHANGES (links).txt'):
+    changes_file = open('CHANGES (links).txt')
+else:
+    # but if the release script has not run, fall back to the source file
+    changes_file = open('CHANGES.txt')
+long_description = readme_file.read() + changes_file.read()
 readme_file.close()
+changes_file.close()
 
 dist = setup(
     name="setuptools",
@@ -144,7 +132,7 @@ dist = setup(
     test_suite = 'setuptools.tests',
     src_root = src_root,
     packages = find_packages(),
-    package_data = {'setuptools':['*.exe']},
+    package_data = {'setuptools':['*.exe'], 'setuptools.command':['*.xml']},
 
     py_modules = ['pkg_resources', 'easy_install', 'site'],
 

setuptools/command/easy_install.py

@@ -19,7 +19,9 @@ import zipfile
 import re
 import stat
 import random
+import platform
 from glob import glob
+import pkg_resources
 from setuptools import Command, _dont_write_bytecode
 from setuptools.sandbox import run_setup
 from distutils import log, dir_util
@@ -522,6 +524,10 @@ Please make the appropriate changes for your system and try again.
         """Write all the scripts for `dist`, unless scripts are excluded"""
         if not self.exclude_scripts and dist.metadata_isdir('scripts'):
             for script_name in dist.metadata_listdir('scripts'):
+                if dist.metadata_isdir('scripts/' + script_name):
+                    # The "script" is a directory, likely a Python 3
+                    # __pycache__ directory, so skip it.
+                    continue
                 self.install_script(
                     dist, script_name,
                     dist.get_metadata('scripts/'+script_name)
@@ -1833,6 +1839,8 @@ def get_script_args(dist, executable=sys_executable, wininst=False):
                     ext, launcher = '-script.py', 'cli.exe'
                     old = ['.py','.pyc','.pyo']
                     new_header = re.sub('(?i)pythonw.exe','python.exe',header)
+                if platform.machine().lower()=='arm':
+                    launcher = launcher.replace(".", "-arm.")
                 if is_64bit():
                     launcher = launcher.replace(".", "-64.")
                 else:
@@ -1846,11 +1854,26 @@ def get_script_args(dist, executable=sys_executable, wininst=False):
                     name+'.exe', resource_string('setuptools', launcher),
                     'b' # write in binary mode
                 )
+                if not is_64bit():
+                    # install a manifest for the launcher to prevent Windows
+                    #  from detecting it as an installer (which it will for
+                    #  launchers like easy_install.exe). Consider only
+                    #  adding a manifest for launchers detected as installers.
+                    #  See Distribute #143 for details.
+                    m_name = name + '.exe.manifest'
+                    yield (m_name, load_launcher_manifest(name), 't')
             else:
                 # On other platforms, we assume the right thing to do is to
                 # just write the stub with no extension.
                 yield (name, header+script_text)
 
+def load_launcher_manifest(name):
+    manifest = pkg_resources.resource_string(__name__, 'launcher manifest.xml')
+    if sys.version_info[0] < 3:
+        return manifest % vars()
+    else:
+        return manifest.decode('utf-8') % vars()
+
 def rmtree(path, ignore_errors=False, onerror=auto_chmod):
     """Recursively delete a directory tree.
 

setuptools/command/launcher manifest.xml

+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
+<assemblyIdentity version="1.0.0.0"
+    processorArchitecture="X86"
+    name="%(name)s"
+    type="win32"/>
+    <!-- Identify the application security requirements. -->
+    <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
+    <security>
+        <requestedPrivileges>
+            <requestedExecutionLevel level="asInvoker" uiAccess="false"/>
+        </requestedPrivileges>
+    </security>
+    </trustInfo>
+</assembly>

setuptools/tests/win_script_wrapper.txt

@@ -38,7 +38,7 @@ We'll also copy cli.exe to the sample-directory with the name foo.exe:
     >>> import pkg_resources
     >>> f = open(os.path.join(sample_directory, 'foo.exe'), 'wb')
     >>> f.write(
-    ...     pkg_resources.resource_string('setuptools', 'cli.exe')
+    ...     pkg_resources.resource_string('setuptools', 'cli-32.exe')
     ...     )
     >>> f.close()
 
@@ -126,7 +126,7 @@ We'll also copy gui.exe to the sample-directory with the name bar.exe:
     >>> import pkg_resources
     >>> f = open(os.path.join(sample_directory, 'bar.exe'), 'wb')
     >>> f.write(
-    ...     pkg_resources.resource_string('setuptools', 'gui.exe')
+    ...     pkg_resources.resource_string('setuptools', 'gui-32.exe')
     ...     )
     >>> f.close()
 

tests/test_pkg_resources.py

+import sys
+import tempfile
+import os
+import zipfile
+
+import pkg_resources
+
+class EggRemover(unicode):
+	def __call__(self):
+		if self in sys.path:
+			sys.path.remove(self)
+		if os.path.exists(self):
+			os.remove(self)
+
+class TestZipProvider(object):
+	finalizers = []
+
+	@classmethod
+	def setup_class(cls):
+		"create a zip egg and add it to sys.path"
+		egg = tempfile.NamedTemporaryFile(suffix='.egg', delete=False)
+		zip_egg = zipfile.ZipFile(egg, 'w')
+		zip_info = zipfile.ZipInfo()
+		zip_info.filename = 'mod.py'
+		zip_info.date_time = 2013, 5, 12, 13, 25, 0
+		zip_egg.writestr(zip_info, 'x = 3\n')
+		zip_info = zipfile.ZipInfo()
+		zip_info.filename = 'data.dat'
+		zip_info.date_time = 2013, 5, 12, 13, 25, 0
+		zip_egg.writestr(zip_info, 'hello, world!')
+		zip_egg.close()
+		egg.close()
+
+		sys.path.append(egg.name)
+		cls.finalizers.append(EggRemover(egg.name))
+
+	@classmethod
+	def teardown_class(cls):
+		for finalizer in cls.finalizers:
+			finalizer()
+
+	def test_resource_filename_rewrites_on_change(self):
+		"""
+		If a previous call to get_resource_filename has saved the file, but
+		the file has been subsequently mutated with different file of the
+		same size and modification time, it should not be overwritten on a
+		subsequent call to get_resource_filename.
+		"""
+		import mod
+		manager = pkg_resources.ResourceManager()
+		zp = pkg_resources.ZipProvider(mod)
+		filename = zp.get_resource_filename(manager, 'data.dat')
+		assert os.stat(filename).st_mtime == 1368379500
+		f = open(filename, 'wb')
+		f.write('hello, world?')
+		f.close()
+		os.utime(filename, (1368379500, 1368379500))
+		filename = zp.get_resource_filename(manager, 'data.dat')
+		f = open(filename)
+		assert f.read() == 'hello, world!'
+		manager.cleanup_resources()