merge gary-5 changes

CHANGES.txt

@@ -11,19 +11,17 @@ New Features:
   than zc.recipe.egg (which is still a fully supported, and simpler, way of
   generating scripts and interpreters if you are using a "clean" Python).
 
-  A hopefully slight limitation: in no cases are distributions in your
-  site-packages used to satisfy buildout dependencies.  The
-  site-packages can be used in addition to the dependencies specified in
-  your buildout, and buildout dependencies can override code in your
-  site-packages, but even if your Python's site-packages has the same
-  exact version as specified in your buildout configuration, buildout
-  will still use its own copy.
-
-- Added new function, ``zc.buildout.easy_install.generate_scripts``, to
-  generate scripts and interpreter.  It produces a full-featured
+  (Note that this branch is incomplete in its implementation of this feature:
+  if eggs are in installed in site-packages but you do not want to use
+  site-packages, the eggs will drag in site-packages even if you try to
+  exclude it.  This is addressed in subsequent branches in the series of
+  which this one is a part.)
+
+- Added new function, ``zc.buildout.easy_install.sitepackage_safe_scripts``,
+  to generate scripts and interpreter.  It produces a full-featured
   interpreter (all command-line options supported) and the ability to
-  safely let scripts include site packages.  The ``z3c.recipe.scripts``
-  recipe uses this new function.
+  safely let scripts include site packages, such as with a system
+  Python.  The ``z3c.recipe.scripts`` recipe uses this new function.
 
 - Improve bootstrap.
 

README.txt

@@ -35,12 +35,15 @@ Existing recipes include:
 `zc.recipe.egg <http://pypi.python.org/pypi/zc.recipe.egg>`_
    The egg recipe installes one or more eggs, with their
    dependencies.  It installs their console-script entry points with
-   the needed eggs included in their paths.
+   the needed eggs included in their paths.  It is suitable for use with
+   a "clean" Python: one without packages installed in site-packages.
 
 `z3c.recipe.scripts <http://pypi.python.org/pypi/z3c.recipe.scripts>`_
-  This scripts recipe builds interpreter scripts and entry point scripts
-  based on eggs.  These scripts have more features and flexibility than the
-  ones offered by zc.recipe.egg.
+  Like zc.recipe.egg, this recipe builds interpreter scripts and entry
+  point scripts based on eggs.  It can be used with a Python that has
+  packages installed in site-packages, such as a system Python.  The
+  interpreter also has more features than the one offered by
+  zc.recipe.egg.
 
 `zc.recipe.testrunner <http://pypi.python.org/pypi/zc.recipe.testrunner>`_
    The testrunner egg creates a test runner script for one or

src/zc/buildout/easy_install.py

@@ -61,15 +61,16 @@ setuptools_loc = pkg_resources.working_set.find(
     pkg_resources.Requirement.parse('setuptools')
     ).location
 
-# Include buildout and setuptools eggs in paths
-buildout_and_setuptools_path = [
-    setuptools_loc,
-    pkg_resources.working_set.find(
-        pkg_resources.Requirement.parse('zc.buildout')).location,
-    ]
+# Include buildout and setuptools eggs in paths.  We prevent dupes just to
+# keep from duplicating any log messages about them.
+buildout_loc = pkg_resources.working_set.find(
+    pkg_resources.Requirement.parse('zc.buildout')).location
+buildout_and_setuptools_path = [setuptools_loc]
+if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
+    buildout_and_setuptools_path.append(buildout_loc)
 
 def _get_system_paths(executable):
-    """return lists of standard lib and site paths for executable.
+    """Return lists of standard lib and site paths for executable.
     """
     # We want to get a list of the site packages, which is not easy.
     # The canonical way to do this is to use
@@ -227,24 +228,47 @@ else:
 #
 # The namespace packages installed in site-packages with
 # --single-version-externally-managed use a mechanism that cause them to
-# be processed when site.py is imported.  Simply starting Python with -S
-# addresses the problem in Python 2.4 and 2.5, but Python 2.6's distutils
-# imports a value from the site module, so we unfortunately have to do more
-# drastic surgery in the _easy_install_cmd code below.  The changes to
-# sys.modules specifically try to only remove namespace modules installed by
-# the --single-version-externally-managed code.
+# be processed when site.py is imported  (see
+# http://mail.python.org/pipermail/distutils-sig/2009-May/011730.html
+# for another description of the problem).  Simply starting Python with
+# -S addresses the problem in Python 2.4 and 2.5, but Python 2.6's
+# distutils imports a value from the site module, so we unfortunately
+# have to do more drastic surgery in the _easy_install_cmd code below.
+#
+# Here's an example of the .pth files created by setuptools when using that
+# flag:
+#
+# import sys,new,os;
+# p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('<NAMESPACE>',));
+# ie = os.path.exists(os.path.join(p,'__init__.py'));
+# m = not ie and sys.modules.setdefault('<NAMESPACE>',new.module('<NAMESPACE>'));
+# mp = (m or []) and m.__dict__.setdefault('__path__',[]);
+# (p not in mp) and mp.append(p)
+#
+# The code, below, then, runs under -S, indicating that site.py should
+# not be loaded initially.  It gets the initial sys.path under these
+# circumstances, and then imports site (because Python 2.6's distutils
+# will want it, as mentioned above). It then reinstates the old sys.path
+# value. Then it removes namespace packages (created by the setuptools
+# code above) from sys.modules.  It identifies namespace packages by
+# iterating over every loaded module.  It first looks if there is a
+# __path__, so it is a package; and then it sees if that __path__ does
+# not have an __init__.py.  (Note that PEP 382,
+# http://www.python.org/dev/peps/pep-0382, makes it possible to have a
+# namespace package that has an __init__.py, but also should make it
+# unnecessary for site.py to preprocess these packages, so it should be
+# fine, as far as can be guessed as of this writing.)  Finally, it
+# imports easy_install and runs it.
 
 _easy_install_cmd = _safe_arg('''\
-import sys; \
-p = sys.path[:]; \
-m = sys.modules.keys(); \
-import site; \
-sys.path[:] = p; \
-m_attrs = set(('__builtins__', '__file__', '__package__', '__path__')); \
-match = set(('__path__',)); \
+import sys,os;\
+p = sys.path[:];\
+import site;\
+sys.path[:] = p;\
 [sys.modules.pop(k) for k, v in sys.modules.items()\
- if k not in m and v and m_attrs.intersection(dir(v)) == match]; \
-from setuptools.command.easy_install import main; \
+ if hasattr(v, '__path__') and len(v.__path__)==1 and\
+ not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];\
+from setuptools.command.easy_install import main;\
 main()''')
 
 
@@ -1126,8 +1150,9 @@ def scripts(reqs, working_set, executable, dest,
             ):
     """Generate scripts and/or an interpreter.
 
-    See generate_scripts for a newer version with more options and a
-    different approach.
+    See sitepackage_safe_scripts for a version that can be used with a Python
+    that can be used with a Python that has code installed in site-packages.
+    It has more options and a different approach.
     """
     path = _get_path(working_set, extra_paths)
     if initialization:
@@ -1142,12 +1167,12 @@ def scripts(reqs, working_set, executable, dest,
             _pyscript(spath, sname, executable, rpsetup))
     return generated
 
-def generate_scripts(
+def sitepackage_safe_scripts(
     dest, working_set, executable, site_py_dest,
     reqs=(), scripts=None, interpreter=None, extra_paths=(),
-    initialization='', add_site_packages=False, exec_sitecustomize=False,
+    initialization='', include_site_packages=False, exec_sitecustomize=False,
     relative_paths=False, script_arguments='', script_initialization=''):
-    """Generate scripts and/or an interpreter.
+    """Generate scripts and/or an interpreter from a system Python.
 
     This accomplishes the same job as the ``scripts`` function, above,
     but it does so in an alternative way that allows safely including
@@ -1159,9 +1184,9 @@ def generate_scripts(
         site_py_dest, executable, initialization, exec_sitecustomize))
     generated.append(_generate_site(
         site_py_dest, working_set, executable, extra_paths,
-        add_site_packages, relative_paths))
+        include_site_packages, relative_paths))
     script_initialization = (
-        '\nimport site # imports custom buildbot-generated site.py\n%s' % (
+        '\nimport site # imports custom buildout-generated site.py\n%s' % (
             script_initialization,))
     if not script_initialization.endswith('\n'):
         script_initialization += '\n'
@@ -1175,7 +1200,7 @@ def generate_scripts(
 
 # Utilities for the script generation functions.
 
-# These are shared by both ``scripts`` and ``generate_scripts``
+# These are shared by both ``scripts`` and ``sitepackage_safe_scripts``
 
 def _get_path(working_set, extra_paths=()):
     """Given working set and extra paths, return a normalized path list."""
@@ -1442,7 +1467,7 @@ if _interactive:
     __import__("code").interact(banner="", local=globals())
 '''
 
-# These are used only by the newer ``generate_scripts`` function.
+# These are used only by the newer ``sitepackage_safe_scripts`` function.
 
 def _get_module_file(executable, name):
     """Return a module's file path.
@@ -1496,10 +1521,10 @@ def _generate_sitecustomize(dest, executable, initialization='',
     return sitecustomize_path
 
 def _generate_site(dest, working_set, executable, extra_paths=(),
-                   add_site_packages=False, relative_paths=False):
+                   include_site_packages=False, relative_paths=False):
     """Write a site.py file with eggs from working_set.
 
-    extra_paths will be added to the path.  If add_site_packages is True,
+    extra_paths will be added to the path.  If include_site_packages is True,
     paths from the underlying Python will be added.
     """
     path = _get_path(working_set, extra_paths)
@@ -1511,7 +1536,7 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
             [(line and '    %s' % (line,) or line)
              for line in preamble.split('\n')])
     original_path_setup = ''
-    if add_site_packages:
+    if include_site_packages:
         stdlib, site_paths = _get_system_paths(executable)
         original_path_setup = original_path_snippet % (
             _format_paths((repr(p) for p in site_paths), 2),)
@@ -1526,7 +1551,7 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
                     relative_paths)
             else:
                 location = repr(distribution.location)
-            preamble += namespace_add_site_packages_setup % (location,)
+            preamble += namespace_include_site_packages_setup % (location,)
             original_path_setup = (
                 addsitedir_namespace_originalpackages_snippet +
                 original_path_setup)
@@ -1555,7 +1580,7 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
         raise RuntimeError('Buildout did not successfully rewrite site.py')
     return site_path
 
-namespace_add_site_packages_setup = '''
+namespace_include_site_packages_setup = '''
     setuptools_path = %s
     sys.path.append(setuptools_path)
     known_paths.add(os.path.normcase(setuptools_path))

src/zc/buildout/tests.py

@@ -1881,8 +1881,9 @@ def handle_namespace_package_in_both_site_packages_and_buildout_eggs():
     r"""
 If you have the same namespace package in both site-packages and in
 buildout, we need to be very careful that faux-Python-executables and
-scripts generated by easy_install.generate_scripts correctly combine the two.
-We show this with the local recipe that uses the function, z3c.recipe.scripts.
+scripts generated by easy_install.sitepackage_safe_scripts correctly
+combine the two. We show this with the local recipe that uses the
+function, z3c.recipe.scripts.
 
 To demonstrate this, we will create three packages: tellmy.version 1.0,
 tellmy.version 1.1, and tellmy.fortune 1.0.  tellmy.version 1.1 is installed.
@@ -1911,7 +1912,7 @@ tellmy.version and tellmy.fortune.
     ... recipe = z3c.recipe.scripts
     ... python = primed_python
     ... interpreter = py
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... eggs = tellmy.version == 1.0
     ...        tellmy.fortune == 1.0
     ...        demo
@@ -1936,7 +1937,7 @@ tellmy.version and tellmy.fortune.
     Generated interpreter '/sample-buildout/bin/py'.
     <BLANKLINE>
 
-Finally, we are ready for the actual test.  Prior to the bug fix that
+Finally, we are ready to see if it worked.  Prior to the bug fix that
 this tests, the results of both calls below was the following::
 
     1.1
@@ -2049,8 +2050,9 @@ Before the bugfix, running this buildout would generate this error:
     We already have: tellmy.version 1.0
     <BLANKLINE>
 
-The bugfix was simply to add Python's "-S" option when calling
-easyinstall (see zc.buildout.easy_install.Installer._call_easy_install).
+You can see the copiously commented fix for this in easy_install.py (see
+zc.buildout.easy_install.Installer._call_easy_install and particularly
+the comment leading up to zc.buildout.easy_install._easy_install_cmd).
 Now the install works correctly, as seen here.
 
     >>> print system(buildout)

src/zc/buildout/testselectingpython.py

@@ -61,7 +61,7 @@ def multi_python(test):
             ['setuptools'], executable_dir,
             index='http://www.python.org/pypi/',
             always_unzip=True, executable=other_executable)
-        zc.buildout.easy_install.generate_scripts(
+        zc.buildout.easy_install.sitepackage_safe_scripts(
             executable_dir, ws, other_executable, executable_parts,
             reqs=['setuptools'], interpreter='py')
         original_executable = other_executable

z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt

@@ -33,7 +33,7 @@ this, we'll list the new options and describe them.
 In addition to these, the recipe offers these new options.  They are
 introduced here, and described more in depth below.
 
-add-site-packages
+include-site-packages
     You can choose to have the site-packages of the underlying Python
     available to your script or interpreter, in addition to the packages
     from your eggs.  See the section on this option for motivations and
@@ -56,16 +56,16 @@ allowed-eggs-from-site-packages
             bigdemo
             zope.*
 
-    This option interacts with the ``add-site-packages`` option in the
+    This option interacts with the ``include-site-packages`` option in the
     following ways.
 
-    If ``add-site-packages`` is true, then
+    If ``include-site-packages`` is true, then
     ``allowed-eggs-from-site-packages`` filters what eggs from site-packages
     may be chosen.  Therefore, if ``allowed-eggs-from-site-packages`` is an
     empty list, then no eggs from site-packages are chosen, but site-packages
     will still be included at the end of path lists.
 
-    If ``add-site-packages`` is false, the value of
+    If ``include-site-packages`` is false, the value of
     ``allowed-eggs-from-site-packages`` is irrelevant.
 
 extends
@@ -76,7 +76,7 @@ extends
 exec-sitecustomize
     Normally the Python's real sitecustomize module is not processed.
     If you want it to be processed, set this value to 'true'.  This will
-    be honored irrespective of the setting for include-site-paths.
+    be honored irrespective of the setting for include-site-packages.
 
 script-initialization
     The standard initialization code affects both an interpreter and scripts.
@@ -210,7 +210,7 @@ some advantages and some serious dangers.
 
 A typical reason to include site-packages is that it is easier to
 install one or more dependencies in your Python than it is with
-buildbot.  Some packages, such as lxml or Python PostgreSQL integration,
+buildout.  Some packages, such as lxml or Python PostgreSQL integration,
 have dependencies that can be much easier to build and/or install using
 other mechanisms, such as your operating system's package manager.  By
 installing some core packages into your Python's site-packages, this can
@@ -231,7 +231,7 @@ instance, it is a system Python), you open yourself up to these
 possibilities.  Don't be unaware of the dangers.
 
 To show off these features, we need to use buildout with a Python
-executable with some extra paths to show ``add-site-packages``; and one
+executable with some extra paths to show ``include-site-packages``; and one
 guaranteed to have a sitecustomize module to show
 ``exec-sitecustomize``.  We'll make one using a test fixture called
 ``make_py``. The os.environ change below will go into the sitecustomize,
@@ -244,7 +244,7 @@ and the site_packages_path will be in the Python's path.
     >>> print site_packages_path
     /executable_buildout/site-packages
 
-Now let's take a look at add-site-packages.
+Now let's take a look at include-site-packages.
 
     >>> write(sample_buildout, 'buildout.cfg',
     ... """
@@ -254,7 +254,7 @@ Now let's take a look at add-site-packages.
     ...
     ... [py]
     ... recipe = z3c.recipe.scripts:interpreter
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... eggs = demo<0.3
     ... find-links = %(server)s
     ... index = %(server)s/index
@@ -428,5 +428,4 @@ interpreter, so that you are not forced to use the name of the section.
     42
     <BLANKLINE>
 
-The other options all identical to the zc.recipe.egg script.  Here are some
-quick demos and discussions.
+The other options all identical to zc.recipe.egg.

z3c.recipe.scripts_/src/z3c/recipe/scripts/scripts.py

@@ -36,13 +36,13 @@ class Base(ScriptBase):
         self.allowed_eggs = tuple(name.strip() for name in value.split('\n'))
 
         value = options.setdefault(
-            'add-site-packages',
-            b_options.get('add-site-packages', 'false'))
+            'include-site-packages',
+            b_options.get('include-site-packages', 'false'))
         if value not in ('true', 'false'):
             raise zc.buildout.UserError(
-                "Invalid value for add-site-packages option: %s" %
+                "Invalid value for include-site-packages option: %s" %
                 (value,))
-        self.add_site_packages = (value == 'true')
+        self.include_site_packages = (value == 'true')
 
         value = options.setdefault(
             'exec-sitecustomize',
@@ -68,13 +68,13 @@ class Interpreter(Base):
         if not os.path.exists(options['parts-directory']):
             os.mkdir(options['parts-directory'])
             generated.append(options['parts-directory'])
-        generated.extend(zc.buildout.easy_install.generate_scripts(
+        generated.extend(zc.buildout.easy_install.sitepackage_safe_scripts(
             options['bin-directory'], ws, options['executable'],
             options['parts-directory'],
             interpreter=options['name'],
             extra_paths=self.extra_paths,
             initialization=options.get('initialization', ''),
-            add_site_packages=self.add_site_packages,
+            include_site_packages=self.include_site_packages,
             exec_sitecustomize=self.exec_sitecustomize,
             relative_paths=self._relative_paths,
             ))
@@ -91,13 +91,13 @@ class Scripts(Base):
         if not os.path.exists(options['parts-directory']):
             os.mkdir(options['parts-directory'])
             generated.append(options['parts-directory'])
-        generated.extend(zc.buildout.easy_install.generate_scripts(
+        generated.extend(zc.buildout.easy_install.sitepackage_safe_scripts(
             options['bin-directory'], ws, options['executable'],
             options['parts-directory'], reqs=reqs, scripts=scripts,
             interpreter=options.get('interpreter'),
             extra_paths=self.extra_paths,
             initialization=options.get('initialization', ''),
-            add_site_packages=self.add_site_packages,
+            include_site_packages=self.include_site_packages,
             exec_sitecustomize=self.exec_sitecustomize,
             relative_paths=self._relative_paths,
             script_arguments=options.get('arguments', ''),

z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py

@@ -237,15 +237,15 @@ Let's look at the site.py that was generated:
             ]...
 """
 
-def add_site_packages_option_reusing_eggs():
+def include_site_packages_option_reusing_eggs():
     """
-The add-site-packages buildout option not only controls whether
+The include-site-packages buildout option not only controls whether
 site-packages are included in the path, but whether eggs in site-packages
 can be used to fulfill direct and indirect dependencies of your package.  If
 it did not, it might fail to exclude site-packages because one of the
 dependencies actually was supposed to be fulfilled with it.
 
-The default is ``add-site-packages = false``.  This makes it possible to
+The default is ``include-site-packages = false``.  This makes it possible to
 easily use a system Python.  As a demonstration, we will start with a
 Python executable that has the "demoneeded" and "demo" eggs installed.
 The eggs are not found.
@@ -277,7 +277,7 @@ The eggs are not found.
     Error: Couldn't find a distribution for 'demoneeded'.
     <BLANKLINE>
 
-However, if we set add-site-packages to true, the package will be found.
+However, if we set include-site-packages to true, the package will be found.
 Notice we do not set find-links, but the eggs are still found because
 they are in the executable's path.
 
@@ -293,7 +293,7 @@ they are in the executable's path.
     ... [eggs]
     ... recipe = z3c.recipe.scripts
     ... python = primed_python
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... eggs = demoneeded
     ... ''' % globals())
 
@@ -311,7 +311,7 @@ We get an error if we specify anything but true or false:
     ...
     ... [eggs]
     ... recipe = z3c.recipe.scripts
-    ... add-site-packages = no
+    ... include-site-packages = no
     ... eggs = other
     ... ''' % globals())
 
@@ -320,7 +320,7 @@ We get an error if we specify anything but true or false:
       Installing.
       Getting section eggs.
       Initializing part eggs.
-    Error: Invalid value for add-site-packages option: no
+    Error: Invalid value for include-site-packages option: no
     <BLANKLINE>
 
     """
@@ -351,7 +351,7 @@ correctly parse a single-line value.
     ...
     ... [eggs]
     ... recipe = z3c.recipe.scripts
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... allowed-eggs-from-site-packages = *
     ... python = primed_python
     ... eggs = demoneeded
@@ -376,7 +376,7 @@ parse a multi-line value.
     ...
     ... [eggs]
     ... recipe = z3c.recipe.scripts
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... allowed-eggs-from-site-packages = other
     ...                                   demoneeded
     ... python = primed_python
@@ -414,7 +414,7 @@ allowed, because we see that we were unable to get "other".
     ...
     ... [eggs]
     ... recipe = z3c.recipe.scripts
-    ... add-site-packages = true
+    ... include-site-packages = true
     ... allowed-eggs-from-site-packages =
     ... eggs = demoneeded
     ... ''' % globals())

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -22,7 +22,7 @@ import zc.buildout.easy_install
 
 class Eggs(object):
 
-    add_site_packages = allowed_eggs = None
+    include_site_packages = allowed_eggs = None
 
     def __init__(self, buildout, name, options):
         self.buildout = buildout
@@ -78,7 +78,7 @@ class Eggs(object):
                 distributions, options['executable'],
                 [options['develop-eggs-directory'],
                  options['eggs-directory']],
-                include_site_packages=self.add_site_packages,
+                include_site_packages=self.include_site_packages,
                 allowed_eggs_from_site_packages=self.allowed_eggs,
                 )
         else:
@@ -92,7 +92,7 @@ class Eggs(object):
                 executable=options['executable'],
                 path=[options['develop-eggs-directory']],
                 newest=b_options.get('newest') == 'true',
-                include_site_packages=self.add_site_packages,
+                include_site_packages=self.include_site_packages,
                 allowed_eggs_from_site_packages=self.allowed_eggs,
                 allow_hosts=self.allow_hosts,
                 **kw)