[fix] Fix working set sorting

If a dist in the computed working set is at a location shared with
other dists - such as site-packages - then when generating scripts
these other packages may overshadow the next items on the sys.path
and result in importing a different version than the one installed
and intended by buildout.

To avert this, a sort of the working set was introduced at various
points just before generating a script.

However, that sort put the paths referenced from an `.egg-link` in
./develop-eggs first. This is truly problematic because dists from
site-packages which are not eggs - e.g. dists installed with pip -
can become referenced as `.egg-link` during buildout bootstrap and
the sort then causes site-packages to be one of the first items in
sys.path.

In particular when running buildout bootstrap from a venv in which
zc.buildout was installed by pip, if any one of zc.buildout or its
dependencies from the venv meets the version requirements, then it
can cause the generated bin/buildout to import the dists only from
the venv's site-packages, even when some do not meet requirements.

To fix this, the sort now puts the dists from `./eggs` first as we
know their locations contain only a single dist, and then puts the
dists from ./develop-eggs which have locations inside the buildout
directory before the others.

The previous sort was also activating all the dists from the paths
of the already activated dists.

Note that this also means that any working set must be manipulated
with care in general to avoid activating unintended dists from the
locations of the already activated dists.

src/zc/buildout/buildout.py

@@ -658,10 +658,12 @@ class Buildout(DictMixin):
         ws = pkg_resources.WorkingSet(entries)
         ws.require('zc.buildout')
         options = self['buildout']
+        buildout_dir = options['directory']
         eggs_dir = options['eggs-directory']
         develop_eggs_dir = options['develop-eggs-directory']
         ws = zc.buildout.easy_install.sort_working_set(
                 ws,
+                buildout_dir=buildout_dir,
                 eggs_dir=eggs_dir,
                 develop_eggs_dir=develop_eggs_dir
                 )
@@ -1204,10 +1206,12 @@ class Buildout(DictMixin):
         # the new dist is different, so we've upgraded.
         # Update the scripts and return True
         options = self['buildout']
+        buildout_dir = options['directory']
         eggs_dir = options['eggs-directory']
         develop_eggs_dir = options['develop-eggs-directory']
         ws = zc.buildout.easy_install.sort_working_set(
                 ws,
+                buildout_dir=buildout_dir,
                 eggs_dir=eggs_dir,
                 develop_eggs_dir=develop_eggs_dir
                 )

src/zc/buildout/easy_install.py

@@ -1958,7 +1958,7 @@ def _move_to_eggs_dir_and_compile(dist, dest):
     return newdist
 
 
-def sort_working_set(ws, eggs_dir, develop_eggs_dir):
+def get_develop_paths(develop_eggs_dir):
     develop_paths = set()
     pattern = os.path.join(develop_eggs_dir, '*.egg-link')
     for egg_link in glob.glob(pattern):
@@ -1966,21 +1966,30 @@ def sort_working_set(ws, eggs_dir, develop_eggs_dir):
             path = f.readline().strip()
             if path:
                 develop_paths.add(path)
+    return develop_paths
 
-    sorted_paths = []
-    egg_paths = []
-    other_paths = []
+
+def sort_working_set(ws, buildout_dir, eggs_dir, develop_eggs_dir):
+    develop_paths = get_develop_paths(develop_eggs_dir)
+    dists_priorities = tuple([] for i in range(5))
     for dist in ws:
         path = dist.location
-        if path in develop_paths:
-            sorted_paths.append(path)
-        elif os.path.commonprefix([path, eggs_dir]) == eggs_dir:
-            egg_paths.append(path)
+        if os.path.commonprefix([path, eggs_dir]) == eggs_dir:
+            # Dists from eggs first because we know they contain a single dist.
+            priority = 0
+        if os.path.commonprefix([path, buildout_dir]) == buildout_dir:
+            # We assume internal locations contain a single dist too.
+            priority = 1 + 2 * (path not in develop_paths) # 1 or 3
         else:
-            other_paths.append(path)
-    sorted_paths.extend(egg_paths)
-    sorted_paths.extend(other_paths)
-    return pkg_resources.WorkingSet(sorted_paths)
+            priority = 2 + 2 * (path not in develop_paths) # 2 or 4
+        dists_priorities[priority].append(dist)
+    # We add dists to an empty working set manually instead of adding the paths
+    # to avoid activating other dists at the same locations.
+    ws = pkg_resources.WorkingSet([])
+    for dists in dists_priorities:
+        for dist in dists:
+            ws.add(dist)
+    return ws
 
 
 NOT_PICKED_AND_NOT_ALLOWED = """\

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -51,6 +51,7 @@ class Eggs(object):
                             if host.strip() != ''])
         self.allow_hosts = allow_hosts
 
+        self.buildout_dir = b_options['directory']
         options['eggs-directory'] = b_options['eggs-directory']
         options['_e'] = options['eggs-directory']  # backward compat.
         options['develop-eggs-directory'] = b_options['develop-eggs-directory']
@@ -77,6 +78,7 @@ class Eggs(object):
             distributions=orig_distributions + list(extra),
             develop_eggs_dir=options['develop-eggs-directory'],
             eggs_dir=options['eggs-directory'],
+            buildout_dir=self.buildout_dir,
             offline=(buildout_section.get('offline') == 'true'),
             newest=(buildout_section.get('newest') == 'true'),
             links=self.links,
@@ -98,6 +100,7 @@ class Eggs(object):
         distributions,
         eggs_dir,
         develop_eggs_dir,
+        buildout_dir,
         offline=False,
         newest=True,
         links=(),
@@ -140,7 +143,7 @@ class Eggs(object):
                     allow_hosts=allow_hosts,
                     allow_unknown_extras=allow_unknown_extras)
             ws = zc.buildout.easy_install.sort_working_set(
-                ws, eggs_dir, develop_eggs_dir
+                ws, buildout_dir, eggs_dir, develop_eggs_dir
                 )
             cache_storage[cache_key] = ws
 

zc.recipe.egg_/src/zc/recipe/egg/working_set_caching.rst

@@ -27,6 +27,7 @@ Here's an example:
     ...     distributions=['zc.recipe.egg', 'demo<0.3'],
     ...     eggs_dir=eggs_dir,
     ...     develop_eggs_dir=develop_eggs_dir,
+    ...     buildout_dir=sample_buildout,
     ...     index=link_server,
     ... )
     Getting...
@@ -52,6 +53,7 @@ built only once.
     ...     distributions=['demo>=0.1'],
     ...     eggs_dir=eggs_dir,
     ...     develop_eggs_dir=develop_eggs_dir,
+    ...     buildout_dir=sample_buildout,
     ...     offline=True,
     ... )
     >>> ws_args_2 = dict(ws_args_1)