Commit 3b16f5af authored by Xavier Thompson's avatar Xavier Thompson

zc.buildout 3.0.1+slapos001: Rebase on zc.buildout 3.0.1

See merge request nexedi/slapos.buildout!30
parents ac3f5e4c d8f72f75
Pipeline #34410 canceled with stage
in 0 seconds
......@@ -28,11 +28,15 @@ The recommended way to install Buildout is to use pip within a virtual environme
.. code-block:: console
virtualenv mybuildout
cd mybuildout
bin/pip install zc.buildout
python3 -m venv myenv
source myenv/bin/activate
pip install zc.buildout
Or for the code from master branch:
.. code-block:: console
pip install https://lab.nexedi.com/nexedi/slapos.buildout/-/archive/master/slapos.buildout-master.tar.gz
To use Buildout, you need to provide a Buildout configuration. Here is
a minimal configuration:
......@@ -98,6 +102,33 @@ specified using *parts*. The parts to be built are listed in the
name that specifies the software to build the part and provides
parameters to control how the part is built.
Bootstrapping an isolated environment
=====================================
Sometimes it is useful to install ``zc.buildout`` and its dependencies
directly in ``eggs`` directory and to generate a ``buildout`` script in
the ``bin`` directory that uses the version in ``eggs`` directory,
instead of relying on the package available in the environment.
One way to achieve this uses the ``extra-paths`` option of ``buildout``
section: by setting it to empty value, packages outside of ``eggs``
or ``develop-eggs`` directories will not be considered when looking
for already installed eggs. Then the ``bootstrap`` command will
install ``zc.buildout`` and its dependencies from scratch in ``eggs``.
.. code-block:: console
buildout buildout:extra-paths= bootstrap
After this, the generated ``bin/buildout`` script will use the packages
installed in ``eggs`` directory instead of those in the environment and
preserve the isolation from the environment, even without setting
``extra-paths``. That is because the default value for ``extra-paths``
only considers the paths where ``zc.buildout`` and its dependencies are
found, and in this case that is only the ``eggs`` directory.
Installing software
===================
......
......@@ -358,6 +358,19 @@ extends-cache
substitutions, and the result is a relative path, then it will be
interpreted relative to the buildout directory.)
.. _extra-paths-buildout-option
extra-paths, default: 'zc.buildout'
Extra paths to scan for already installed distributions.
Setting this to an empty value enables isolation of buildout.
Setting this to 'legacy' enables the legacy behavior of
scanning the paths of the distributions of zc.buildout itself
and its dependencies, which may contain sites-packages or not.
Setting this to 'zc.buildout' also scans the paths of the
current zc.buildout and dependencies, but respects the order
they appear in sys.path, avoiding unexpected results.
.. _find-links-option:
find-links, default: ''
......
......@@ -12,7 +12,7 @@
#
##############################################################################
name = "zc.buildout"
version = '3.0.1'
version = '3.0.1+slapos001'
import os
from setuptools import setup
......@@ -47,7 +47,7 @@ setup(
python_requires = '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
namespace_packages = ['zc'],
install_requires = [
'setuptools>=8.0',
'setuptools>=38.2.3',
'pip',
'wheel',
],
......
......@@ -49,3 +49,16 @@ class UserError(Exception):
def __str__(self):
return " ".join(map(str, self.args))
# Used for Python 2-3 compatibility
if str is bytes: # BBB Py2
bytes2str = str2bytes = lambda s: s
def unicode2str(s):
return s.encode('utf-8')
else:
def bytes2str(s):
return s.decode()
def str2bytes(s):
return s.encode()
def unicode2str(s):
return s
This diff is collapsed.
......@@ -113,10 +113,12 @@ section_header = re.compile(
r'([#;].*)?$)'
).match
option_name_re = r'[^\s{}[\]=:]+'
option_start = re.compile(
r'(?P<name>[^\s{}[\]=:]+\s*[-+]?)'
r'(?P<name>%s\s*[-+]?)'
r'='
r'(?P<value>.*)$').match
r'(?P<value>.*)$'
% option_name_re).match
leading_blank_lines = re.compile(r"^(\s*\n)+")
......@@ -201,7 +203,12 @@ def parse(fp, fpname, exp_globals=dict):
if not context:
context = exp_globals()
# evaluated expression is in list: get first element
section_condition = eval(expr, context)[0]
try:
section_condition = eval(expr, context)[0]
except NameError as x:
sections.setdefault(sectname, {})[
'__unsupported_conditional_expression__'] = x
continue
# finally, ignore section when an expression
# evaluates to false
if not section_condition:
......@@ -255,6 +262,8 @@ def parse(fp, fpname, exp_globals=dict):
section = sections[sectname]
for name in section:
value = section[name]
if isinstance(value, NameError):
continue
if value[:1].isspace():
section[name] = leading_blank_lines.sub(
'', textwrap.dedent(value.rstrip()))
......
This diff is collapsed.
This diff is collapsed.
......@@ -16,7 +16,8 @@
import shutil
import os
import doctest
import time
import errno
import sys
def rmtree (path):
"""
......@@ -26,6 +27,10 @@ def rmtree (path):
process (e.g. antivirus scanner). This tries to chmod the
file to writeable and retries 10 times before giving up.
Also it tries to remove symlink itself if a symlink as passed as
path argument.
Finally, it tries to make parent directory writable.
>>> from tempfile import mkdtemp
Let's make a directory ...
......@@ -41,13 +46,51 @@ def rmtree (path):
>>> foo = os.path.join (d, 'foo')
>>> with open (foo, 'w') as f: _ = f.write ('huhu')
>>> bar = os.path.join (d, 'bar')
>>> os.symlink(bar, bar)
and make it unwriteable
>>> os.chmod (foo, 256) # 0400
>>> os.chmod (foo, 0o400)
and make parent dir unwritable
>>> os.chmod (d, 0o400)
rmtree should be able to remove it:
>>> rmtree (d)
and now the directory is gone
>>> os.path.isdir (d)
0
Let's make a directory ...
>>> d = mkdtemp()
and make sure it is actually there
>>> os.path.isdir (d)
1
Now create a broken symlink ...
>>> foo = os.path.join (d, 'foo')
>>> os.symlink(foo + '.not_exist', foo)
rmtree should be able to remove it:
>>> rmtree (foo)
and now the directory is gone
>>> os.path.isdir (foo)
0
cleanup directory
>>> rmtree (d)
and now the directory is gone
......@@ -55,20 +98,43 @@ def rmtree (path):
>>> os.path.isdir (d)
0
"""
def retry_writeable (func, path, exc):
os.chmod (path, 384) # 0600
for i in range(10):
try:
func (path)
break
except OSError:
time.sleep(0.1)
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
if func is os.path.islink:
os.unlink(path)
elif func is os.lstat or func is os.open:
if not os.path.islink(path):
raise
os.unlink(path)
else:
# tried 10 times without success, thus
# finally rethrow the last exception
# Depending on the Python version, the following items differ.
if sys.version_info >= (3, ):
expected_error_type = PermissionError
expected_func_tuple = (os.lstat, os.open)
else:
expected_error_type = OSError
expected_func_tuple = (os.listdir, )
e = exc_info[1]
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func in expected_func_tuple:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
return shutil.rmtree(failed_path, onerror=chmod_retry)
# If parent directory is not writable, we still cannot delete the file.
# But make sure not to change the parent of the folder we are deleting.
if failed_path != path:
os.chmod(os.path.dirname(failed_path), 0o700)
return func(failed_path)
raise
shutil.rmtree (path, onerror = retry_writeable)
shutil.rmtree(path, onerror=chmod_retry)
def test_suite():
return doctest.DocTestSuite()
......
......@@ -23,6 +23,7 @@ except ImportError:
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from urllib2 import urlopen
import base64
import errno
import logging
import multiprocessing
......@@ -222,6 +223,9 @@ class Buildout(zc.buildout.buildout.Buildout):
Options = TestOptions
def initialize(self, *args):
pass
def buildoutSetUp(test):
test.globs['__tear_downs'] = __tear_downs = []
......@@ -412,6 +416,23 @@ class Handler(BaseHTTPRequestHandler):
self.__server.__log = False
return k()
if self.path.startswith('/private/'):
auth = self.headers.get('Authorization')
if auth and auth.startswith('Basic ') and \
self.path[9:].encode() == base64.b64decode(
self.headers.get('Authorization')[6:]):
return k()
# But not returning 401+WWW-Authenticate, we check that the client
# skips auth challenge, which is not free (in terms of performance)
# and useless for what we support.
self.send_response(403, 'Forbidden')
out = '<html><body>Forbidden</body></html>'.encode()
self.send_header('Content-Length', str(len(out)))
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(out)
return
path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
if not (
((path == self.tree) or path.startswith(self.tree+os.path.sep))
......@@ -622,6 +643,8 @@ ignore_not_upgrading = (
'Not upgrading because not running a local buildout command.\n'
), '')
os.environ['BUILDOUT_INFO_REINSTALL_REASON'] = '0'
def run_buildout(command):
# Make sure we don't get .buildout
os.environ['HOME'] = os.path.join(os.getcwd(), 'home')
......
......@@ -123,6 +123,45 @@ def create_sample_eggs(test, executable=sys.executable):
)
zc.buildout.testing.bdist_egg(tmp, sys.executable, dest)
write(tmp, 'builddep.py', '')
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='builddep', "
" py_modules=['builddep'], "
" zip_safe=True, version='0.1')\n"
)
zc.buildout.testing.sdist(tmp, dest)
write(tmp, 'withsetuprequires.py', '')
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='withsetuprequires', "
" setup_requires = 'builddep', "
" py_modules=['withsetuprequires'], "
" zip_safe=True, version='0.1')\n"
"import builddep"
)
zc.buildout.testing.sdist(tmp, dest)
write(tmp, 'withbuildsystemrequires.py', '')
write(tmp, 'pyproject.toml',
'[build-system]\n'
'requires = ["builddep"]'
)
write(
tmp, 'setup.py',
"from setuptools import setup\n"
"setup(name='withbuildsystemrequires', "
" setup_requires = 'builddep', "
" py_modules=['withbuildsystemrequires'], "
" package_data={'withbuildsystemrequires': ['pyproject.toml']}, "
" zip_safe=True, version='0.1')\n"
"import builddep"
)
zc.buildout.testing.sdist(tmp, dest)
finally:
shutil.rmtree(tmp)
......
......@@ -40,6 +40,7 @@ Now we can run the buildout and see that it fails:
...
While:
Installing eggs.
Base installation request: 'allowdemo[bad_extra]'
Error: Couldn't find the required extra...
If we flip the option on, the buildout succeeds
......
......@@ -61,6 +61,8 @@ Now we can run the buildout and make sure all attempts to dist.plone.org fails::
...
While:
Installing eggs.
Base installation request: 'allowdemo'
Requirement of allowdemo: kss.core
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
......@@ -92,6 +94,8 @@ Now we can run the buildout and make sure all attempts to dist.plone.org fails::
...
While:
Installing eggs.
Base installation request: 'allowdemo'
Requirement of allowdemo: kss.core
Getting distribution for 'kss.core'.
Error: Couldn't find a distribution for 'kss.core'.
......
......@@ -337,6 +337,10 @@ we'll see that the directory gets removed and recreated::
... path = mydata
... """)
>>> print_(system(buildout+' --dry-run'), end='')
Develop: '/sample-buildout/recipes'
Uninstalling data-dir.
Installing data-dir.
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Uninstalling data-dir.
......@@ -357,6 +361,10 @@ If any of the files or directories created by a recipe are removed,
the part will be reinstalled::
>>> rmdir(sample_buildout, 'mydata')
>>> print_(system(buildout+' --dry-run'), end='')
Develop: '/sample-buildout/recipes'
Uninstalling data-dir.
Installing data-dir.
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Uninstalling data-dir.
......@@ -816,6 +824,8 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
DEFAULT_VALUE
directory= /sample-buildout
COMPUTED_VALUE
dry-run= false
DEFAULT_VALUE
eggs-directory= /sample-buildout/eggs
DEFAULT_VALUE
executable= ...
......@@ -911,6 +921,11 @@ You get more information about the way values are computed::
AS COMPUTED_VALUE
SET VALUE = /sample-buildout
<BLANKLINE>
dry-run= false
<BLANKLINE>
AS DEFAULT_VALUE
SET VALUE = false
<BLANKLINE>
eggs-directory= /sample-buildout/eggs
<BLANKLINE>
AS DEFAULT_VALUE
......@@ -1269,6 +1284,102 @@ the current section. We can also use the special option,
my_name debug
recipe recipes:debug
It is possible to have access to profile base url from section by
using ${:_profile_base_location_}:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = data-dir debug
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
... profile_base_location = ${:_profile_base_location_}
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Uninstalling debug.
Updating data-dir.
Installing debug.
_profile_base_location_ /sample-buildout
profile_base_location /sample-buildout
recipe recipes:debug
Keep in mind that in case of sections spaning across multiple profiles,
the topmost value will be presented:
>>> write(sample_buildout, 'extended.cfg',
... """
... [debug]
... profile_base_location = ${:_profile_base_location_}
... """)
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = extended.cfg
... develop = recipes
... parts = data-dir debug
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
... profile_base_location = ${:_profile_base_location_}
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Updating data-dir.
Updating debug.
_profile_base_location_ /sample-buildout
profile_base_location /sample-buildout
recipe recipes:debug
But of course, in case if accessing happens in extended profile's section,
this profile's location will be exposed:
>>> write(sample_buildout, 'extended.cfg',
... """
... [debug]
... profile_base_location = ${:_profile_base_location_}
... """)
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = extended.cfg
... develop = recipes
... parts = data-dir debug
... log-level = INFO
...
... [debug]
... recipe = recipes:debug
...
... [data-dir]
... recipe = recipes:mkdir
... path = mydata
... """)
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Updating data-dir.
Updating debug.
_profile_base_location_ /sample-buildout
profile_base_location /sample-buildout
recipe recipes:debug
>>> remove(sample_buildout, 'extended.cfg')
Automatic part selection and ordering
-------------------------------------
......@@ -2700,7 +2811,7 @@ were created.
The ``.installed.cfg`` is only updated for the recipes that ran::
>>> cat(sample_buildout, '.installed.cfg')
... # doctest: +NORMALIZE_WHITESPACE
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = debug d1 d2 d3 d4
......@@ -2730,7 +2841,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
<BLANKLINE>
[d4]
__buildout_installed__ = /sample-buildout/data2-extra
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
path = /sample-buildout/data2-extra
recipe = recipes:mkdir
......@@ -2804,10 +2915,10 @@ provide alternate locations, and even names for these directories::
Creating directory '/sample-alt/work'.
Creating directory '/sample-alt/developbasket'.
Develop: '/sample-buildout/recipes'
Uninstalling d4.
Uninstalling d3.
Uninstalling d2.
Uninstalling debug.
Uninstalling d4.
Uninstalling d3.
>>> ls(alt)
d basket
......@@ -2915,8 +3026,10 @@ database is shown::
bin-directory = /sample-buildout/bin
develop-eggs-directory = /sample-buildout/develop-eggs
directory = /sample-buildout
dry-run = false
eggs-directory = /sample-buildout/eggs
executable = python
extra-paths = ...
find-links =
install-from-cache = false
installed = /sample-buildout/.installed.cfg
......@@ -3234,7 +3347,6 @@ or paths to use::
>>> remove('setup.cfg')
>>> print_(system(buildout + ' -csetup.cfg init demo other ./src'), end='')
Creating '/sample-bootstrapped/setup.cfg'.
Creating directory '/sample-bootstrapped/develop-eggs'.
Getting distribution for 'zc.recipe.egg>=2.0.6'.
Got zc.recipe.egg
Installing py.
......@@ -3293,7 +3405,6 @@ for us::
>>> remove('setup.cfg')
>>> print_(system(buildout + ' -csetup.cfg init demo other ./src'), end='')
Creating '/sample-bootstrapped/setup.cfg'.
Creating directory '/sample-bootstrapped/develop-eggs'.
Installing py.
Generated script '/sample-bootstrapped/bin/demo'.
Generated script '/sample-bootstrapped/bin/distutilsscript'.
......
......@@ -87,6 +87,8 @@ buildout to see where the egg comes from this time.
...
While:
Updating eggs.
Base installation request: 'depdemo'
Requirement of depdemo: demoneeded
Getting distribution for 'demoneeded'.
Error: Couldn't find a distribution for 'demoneeded'.
......
......@@ -63,6 +63,32 @@ When trying to access a file that doesn't exist, we'll get an exception:
... else: print_('woops')
download error
An alternate URL can be used in case of HTTPError with the main one.
Useful when a version of a resource can only be downloaded with a temporary
URL as long as it's the last version, and this version is then moved to a
permanent place when a newer version is released. In such case, when using
a cache (in particular networkcache), it's important that the main URL (`url`)
is always used as cache key. And `alternate_url` shall be the temporary URL.
>>> path, is_temp = download(server_url+'not-there',
... alternate_url=server_url+'foo.txt')
>>> cat(path)
This is a foo text.
>>> is_temp
True
>>> remove(path)
The main URL is tried first:
>>> write(server_data, 'other.txt', 'This is some other text.')
>>> path, is_temp = download(server_url+'other.txt',
... alternate_url=server_url+'foo.txt')
>>> cat(path)
This is some other text.
>>> is_temp
True
>>> remove(path)
Downloading a local file doesn't produce a temporary file but simply returns
the local file itself:
......@@ -126,6 +152,37 @@ This is a foo text.
>>> remove(path)
HTTP basic authentication:
>>> download = Download()
>>> user_url = server_url.replace('/localhost:', '/%s@localhost:') + 'private/'
>>> path, is_temp = download(user_url % 'foo:' + 'foo:')
>>> is_temp; remove(path)
True
>>> path, is_temp = download(user_url % 'foo:bar' + 'foo:bar')
>>> is_temp; remove(path)
True
>>> download(user_url % 'bar:' + 'foo:')
Traceback (most recent call last):
UserError: Error downloading ...: HTTP Error 403: Forbidden
... with netrc:
>>> url = server_url + 'private/foo:bar'
>>> download(url)
Traceback (most recent call last):
UserError: Error downloading ...: HTTP Error 403: Forbidden
>>> import os, zc.buildout.download
>>> old_home = os.environ['HOME']
>>> home = os.environ['HOME'] = tmpdir('test-netrc')
>>> netrc = join(home, '.netrc')
>>> write(netrc, 'machine localhost login foo password bar')
>>> os.chmod(netrc, 0o600)
>>> zc.buildout.download.netrc.__init__()
>>> path, is_temp = download(url)
>>> is_temp; remove(path)
True
>>> os.environ['HOME'] = old_home
Downloading using the download cache
------------------------------------
......@@ -165,14 +222,6 @@ the file on the server to see this:
>>> cat(path)
This is a foo text.
If we specify an MD5 checksum for a file that is already in the cache, the
cached copy's checksum will be verified:
>>> download(server_url+'foo.txt', md5('The wrong text.'.encode()).hexdigest())
Traceback (most recent call last):
ChecksumError: MD5 checksum mismatch for cached download
from 'http://localhost/foo.txt' at '/download-cache/foo.txt'
Trying to access another file at a different URL which has the same base name
will result in the cached copy being used:
......@@ -184,6 +233,14 @@ will result in the cached copy being used:
>>> cat(path)
This is a foo text.
If we specify an MD5 checksum for a file that is already in the cache, the
cached copy's checksum will be verified and the cache will be refreshed:
>>> path, is_temp = download(server_url+'foo.txt', md5('The wrong text.'.encode()).hexdigest())
>>> is_temp
True
>>> remove(path)
Given a target path for the download, the utility will provide a copy of the
file at that location both when first downloading the file and when using a
cached copy:
......@@ -259,7 +316,7 @@ If the file is completely missing it should notify the user of the error:
>>> download(server_url+'bar.txt') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Traceback (most recent call last):
...
UserError: Error downloading extends for URL http://localhost/bar.txt:
UserError: Error downloading http://localhost/bar.txt:
...404...
>>> ls(cache)
......@@ -442,18 +499,22 @@ However, when downloading the file normally with the cache being used in
fall-back mode, the file will be downloaded from the net and the cached copy
will be replaced with the new content:
>>> cat(download(server_url+'foo.txt')[0])
>>> path, is_temp = download(server_url+'foo.txt')
>>> cat(path)
The wrong text.
>>> cat(cache, 'foo.txt')
The wrong text.
>>> is_temp
True
>>> remove(path)
When trying to download a resource whose checksum does not match, the cached
copy will neither be used nor overwritten:
Fall-back mode is meaningless if md5sum is given. If the checksum of the
cached copy matches, the resource is not downloaded:
>>> write(server_data, 'foo.txt', 'This is a foo text.')
>>> download(server_url+'foo.txt', md5('The wrong text.'.encode()).hexdigest())
Traceback (most recent call last):
ChecksumError: MD5 checksum mismatch downloading 'http://localhost/foo.txt'
>>> path, is_temp = download(server_url+'foo.txt', md5('The wrong text.'.encode()).hexdigest())
>>> print_(path)
/download-cache/foo.txt
>>> cat(cache, 'foo.txt')
The wrong text.
......
......@@ -33,11 +33,12 @@ download:
>>> print_(get(link_server), end='')
<html><body>
<a href="bigdemo-0.1-py2.4.egg">bigdemo-0.1-py2.4.egg</a><br>
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4rc1-py2.4.egg">demo-0.4rc1-py2.4.egg</a><br>
<a href="bigdemo-0.1-pyN.N.egg">bigdemo-0.1-pyN.N.egg</a><br>
<a href="builddep-0.1.zip">builddep-0.1.zip</a><br>
<a href="demo-0.1-pyN.N.egg">demo-0.1-pyN.N.egg</a><br>
<a href="demo-0.2-pyN.N.egg">demo-0.2-pyN.N.egg</a><br>
<a href="demo-0.3-pyN.N.egg">demo-0.3-pyN.N.egg</a><br>
<a href="demo-0.4rc1-pyN.N.egg">demo-0.4rc1-pyN.N.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2rc1.zip">demoneeded-1.2rc1.zip</a><br>
......@@ -45,7 +46,9 @@ download:
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
<a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
<a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
<a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
</body></html>
......
......@@ -97,11 +97,12 @@ We have a link server that has a number of eggs:
>>> print_(get(link_server), end='')
<html><body>
<a href="bigdemo-0.1-py2.4.egg">bigdemo-0.1-py2.4.egg</a><br>
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4rc1-py2.4.egg">demo-0.4rc1-py2.4.egg</a><br>
<a href="bigdemo-0.1-pyN.N.egg">bigdemo-0.1-pyN.N.egg</a><br>
<a href="builddep-0.1.zip">builddep-0.1.zip</a><br>
<a href="demo-0.1-pyN.N.egg">demo-0.1-pyN.N.egg</a><br>
<a href="demo-0.2-pyN.N.egg">demo-0.2-pyN.N.egg</a><br>
<a href="demo-0.3-pyN.N.egg">demo-0.3-pyN.N.egg</a><br>
<a href="demo-0.4rc1-pyN.N.egg">demo-0.4rc1-pyN.N.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2rc1.zip">demoneeded-1.2rc1.zip</a><br>
......@@ -109,7 +110,9 @@ We have a link server that has a number of eggs:
<a href="extdemo-1.4.zip">extdemo-1.4.zip</a><br>
<a href="index/">index/</a><br>
<a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
<a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
<a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
<a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
</body></html>
Let's make a directory and install the demo egg to it, using the demo:
......@@ -765,9 +768,9 @@ An interpreter can also be generated without other eggs:
<BLANKLINE>
import sys
<BLANKLINE>
sys.path[0:0] = [
<BLANKLINE>
]
<BLANKLINE>
_interactive = True
...
An additional argument can be passed to define which scripts to install
......@@ -1233,11 +1236,12 @@ Let's update our link server with a new version of extdemo:
>>> update_extdemo()
>>> print_(get(link_server), end='')
<html><body>
<a href="bigdemo-0.1-py2.4.egg">bigdemo-0.1-py2.4.egg</a><br>
<a href="demo-0.1-py2.4.egg">demo-0.1-py2.4.egg</a><br>
<a href="demo-0.2-py2.4.egg">demo-0.2-py2.4.egg</a><br>
<a href="demo-0.3-py2.4.egg">demo-0.3-py2.4.egg</a><br>
<a href="demo-0.4rc1-py2.4.egg">demo-0.4rc1-py2.4.egg</a><br>
<a href="bigdemo-0.1-pyN.N.egg">bigdemo-0.1-pyN.N.egg</a><br>
<a href="builddep-0.1.zip">builddep-0.1.zip</a><br>
<a href="demo-0.1-pyN.N.egg">demo-0.1-pyN.N.egg</a><br>
<a href="demo-0.2-pyN.N.egg">demo-0.2-pyN.N.egg</a><br>
<a href="demo-0.3-pyN.N.egg">demo-0.3-pyN.N.egg</a><br>
<a href="demo-0.4rc1-pyN.N.egg">demo-0.4rc1-pyN.N.egg</a><br>
<a href="demoneeded-1.0.zip">demoneeded-1.0.zip</a><br>
<a href="demoneeded-1.1.zip">demoneeded-1.1.zip</a><br>
<a href="demoneeded-1.2rc1.zip">demoneeded-1.2rc1.zip</a><br>
......@@ -1246,7 +1250,9 @@ Let's update our link server with a new version of extdemo:
<a href="extdemo-1.5.zip">extdemo-1.5.zip</a><br>
<a href="index/">index/</a><br>
<a href="mixedcase-0.5.zip">mixedcase-0.5.zip</a><br>
<a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
<a href="other-1.0-pyN.N.egg">other-1.0-pyN.N.egg</a><br>
<a href="withbuildsystemrequires-0.1.zip">withbuildsystemrequires-0.1.zip</a><br>
<a href="withsetuprequires-0.1.zip">withsetuprequires-0.1.zip</a><br>
</body></html>
The easy_install caches information about servers to reduce network
......@@ -1445,9 +1451,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/
GET 200 /index/
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
......@@ -1469,6 +1474,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but
......
......@@ -492,9 +492,9 @@ a better solution would re-use the logging already done by the utility.)
>>> import zc.buildout
>>> old_download = zc.buildout.download.Download.download
>>> def wrapper_download(self, url, md5sum=None, path=None):
>>> def wrapper_download(self, url, *args, **kw):
... print_("The URL %s was downloaded." % url)
... return old_download(url, md5sum, path)
... return old_download(url, *args, **kw)
>>> zc.buildout.download.Download.download = wrapper_download
>>> zc.buildout.buildout.main([])
......
......@@ -207,6 +207,7 @@ versions:
Getting section foo.
Initializing section foo.
Installing recipe spam.
Base installation request: 'spam'
Getting distribution for 'spam'.
Error: Picked: spam = 2
...
......
This diff is collapsed.
......@@ -14,7 +14,7 @@
"""Setup for zc.recipe.egg package
"""
version = '2.0.8.dev0'
version = '2.0.8.dev0+slapos001'
import os
from setuptools import setup, find_packages
......
......@@ -9,6 +9,19 @@ eggs
requirement strings. Each string must be given on a separate
line.
patch-binary
The path to the patch executable.
EGGNAME-patches
A new-line separated list of patchs to apply when building.
EGGNAME-patch-options
Options to give to the patch program when applying patches.
EGGNAME-patch-revision
An integer to specify the revision (default is the number of
patches).
find-links
A list of URLs, files, or directories to search for distributions.
......
......@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
We can see that the options were augmented with additional data
computed by the egg recipe by looking at .installed.cfg:
>>> cat(sample_buildout, '.installed.cfg')
>>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
parts = sample-part
<BLANKLINE>
[sample-part]
__buildout_installed__ =
__buildout_signature__ = ...
__buildout_signature__ = sample-... setuptools-... zc.buildout-... zc.recipe.egg-...
_b = /sample-buildout/bin
_d = /sample-buildout/develop-eggs
_e = /sample-buildout/eggs
......
......@@ -15,6 +15,7 @@
"""
import logging
import os
import re
import sys
import zc.buildout.easy_install
......@@ -28,17 +29,19 @@ class Base:
self.name, self.options = name, options
options['_d'] = buildout['buildout']['develop-eggs-directory']
self.build_ext = build_ext(buildout, options)
def update(self):
return self.install()
options['_e'] = buildout['buildout']['eggs-directory']
class Custom(Base):
environment_section = options.get('environment')
if environment_section:
self.environment = buildout[environment_section]
else:
self.environment = {}
environment_data = list(self.environment.items())
environment_data.sort()
options['_environment-data'] = repr(environment_data)
def __init__(self, buildout, name, options):
Base.__init__(self, buildout, name, options)
self.build_ext = build_ext(buildout, options)
links = options.get('find-links',
buildout['buildout'].get('find-links'))
......@@ -54,45 +57,20 @@ class Custom(Base):
options['index'] = index
self.index = index
environment_section = options.get('environment')
if environment_section:
self.environment = buildout[environment_section]
else:
self.environment = {}
environment_data = list(self.environment.items())
environment_data.sort()
options['_environment-data'] = repr(environment_data)
options['_e'] = buildout['buildout']['eggs-directory']
if buildout['buildout'].get('offline') == 'true':
self.install = lambda: ()
self.newest = buildout['buildout'].get('newest') == 'true'
def install(self):
options = self.options
distribution = options.get('egg')
if distribution is None:
distribution = options.get('eggs')
if distribution is None:
distribution = self.name
else:
logger.warn("The eggs option is deprecated. Use egg instead")
distribution = options.get('egg', options.get('eggs', self.name)
).strip()
def install(self):
self._set_environment()
try:
return zc.buildout.easy_install.build(
distribution, options['_d'], self.build_ext,
self.links, self.index, sys.executable,
[options['_e']], newest=self.newest,
)
self._install_setup_eggs()
return self._install()
finally:
self._restore_environment()
def update(self):
return self.install()
def _set_environment(self):
self._saved_environment = {}
......@@ -114,6 +92,78 @@ class Custom(Base):
except KeyError:
pass
def _install_setup_eggs(self):
options = self.options
setup_eggs = [
r.strip()
for r in options.get('setup-eggs', '').split('\n')
if r.strip()]
if setup_eggs:
ws = zc.buildout.easy_install.install(
setup_eggs, options['_e'],
links=self.links,
index=self.index,
executable=sys.executable,
path=[options['_d'], options['_e']],
newest=self.newest,
)
extra_path = os.pathsep.join(ws.entries)
self.environment['PYTHONEXTRAPATH'] = os.environ['PYTHONEXTRAPATH'] = extra_path
def _get_patch_dict(self, options, distribution):
patch_dict = {}
global_patch_binary = options.get('patch-binary', 'patch')
def get_option(egg, key, default):
return options.get('%s-%s' % (egg, key),
options.get(key, default))
egg = re.sub('[<>=].*', '', distribution)
patches = filter(lambda x:x,
map(lambda x:x.strip(),
get_option(egg, 'patches', '').splitlines()))
patches = list(patches)
if not patches:
return patch_dict
patch_options = get_option(egg, 'patch-options', '-p0').split()
patch_binary = get_option(egg, 'patch-binary', global_patch_binary)
patch_revision = int(get_option(egg, 'patch-revision', len(patches)))
patch_dict[egg] = {
'patches':patches,
'patch_options':patch_options,
'patch_binary':patch_binary,
'patch_revision':patch_revision,
}
return patch_dict
class Custom(Base):
def __init__(self, buildout, name, options):
Base.__init__(self, buildout, name, options)
if buildout['buildout'].get('offline') == 'true':
self._install = lambda: ()
def _install(self):
options = self.options
distribution = options.get('egg')
if distribution is None:
distribution = options.get('eggs')
if distribution is None:
distribution = self.name
else:
logger.warn("The eggs option is deprecated. Use egg instead")
distribution = options.get('egg', options.get('eggs', self.name)
).strip()
patch_dict = self._get_patch_dict(options, distribution)
return zc.buildout.easy_install.build(
distribution, options['_d'], self.build_ext,
self.links, self.index, sys.executable,
[options['_e']], newest=self.newest, patch_dict=patch_dict,
)
class Develop(Base):
......@@ -122,7 +172,7 @@ class Develop(Base):
options['setup'] = os.path.join(buildout['buildout']['directory'],
options['setup'])
def install(self):
def _install(self):
options = self.options
return zc.buildout.easy_install.develop(
options['setup'], options['_d'], self.build_ext)
......
......@@ -20,6 +20,23 @@ rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
setup-eggs
A new-line separated list of eggs that need to be installed
beforehand. It is useful to meet the `setup_requires` requirement.
patch-binary
The path to the patch executable.
patches
A new-line separated list of patchs to apply when building.
patch-options
Options to give to the patch program when applying patches.
patch-revision
An integer to specify the revision (default is the number of
patches).
define
A comma-separated list of names of C preprocessor variables to
define.
......@@ -434,8 +451,8 @@ Create a clean buildout.cfg w/o the checkenv recipe, and delete the recipe:
... """ % dict(server=link_server))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Develop: '/sample-buildout/recipes'
Uninstalling checkenv.
Uninstalling extdemo.
Uninstalling checkenv.
Installing extdemo...
>>> rmdir(sample_buildout, 'recipes')
......@@ -463,6 +480,10 @@ rpath
A new-line separated list of directories to search for dynamic libraries
at run time.
setup-eggs
A new-line separated list of eggs that need to be installed
beforehand. It is useful to meet the `setup_requires` requirement.
define
A comma-separated list of names of C preprocessor variables to
define.
......@@ -499,6 +520,10 @@ swig-cpp
swig-opts
List of SWIG command line options
environment
The name of a section with additional environment variables. The
environment variables are set before the egg is built.
To illustrate this, we'll use a directory containing the extdemo
example from the earlier section:
......
......@@ -51,11 +51,44 @@ class Eggs(object):
if host.strip() != ''])
self.allow_hosts = allow_hosts
self.buildout_dir = b_options['directory']
options['eggs-directory'] = b_options['eggs-directory']
options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory'] = b_options['develop-eggs-directory']
options['_d'] = options['develop-eggs-directory'] # backward compat.
def _get_patch_dict(self, options, egg=None):
patch_dict = {}
global_patch_binary = options.get('patch-binary', 'patch')
if egg:
egg = re.sub('[<>=].*', '', egg)
egg_list = [egg]
else:
egg_list = [x[:-8] for x in options.keys() if x.endswith('-patches')]
def get_option(egg, key, default):
if len(egg_list) == 1:
return options.get('%s-%s' % (egg, key),
options.get(key, default))
else:
return options.get('%s-%s' % (egg, key), default)
for egg in egg_list:
patches = filter(lambda x:x,
map(lambda x:x.strip(),
get_option(egg, 'patches', '').splitlines()))
patches = list(patches)
if not patches:
continue
patch_options = get_option(egg, 'patch-options', '-p0').split()
patch_binary = get_option(egg, 'patch-binary', global_patch_binary)
patch_revision = int(get_option(egg, 'patch-revision', len(patches)))
patch_dict[egg] = {
'patches':patches,
'patch_options':patch_options,
'patch_binary':patch_binary,
'patch_revision':patch_revision,
}
return patch_dict
def working_set(self, extra=()):
"""Separate method to just get the working set
......@@ -77,6 +110,7 @@ class Eggs(object):
distributions=orig_distributions + list(extra),
develop_eggs_dir=options['develop-eggs-directory'],
eggs_dir=options['eggs-directory'],
buildout_dir=self.buildout_dir,
offline=(buildout_section.get('offline') == 'true'),
newest=(buildout_section.get('newest') == 'true'),
links=self.links,
......@@ -98,6 +132,7 @@ class Eggs(object):
distributions,
eggs_dir,
develop_eggs_dir,
buildout_dir,
offline=False,
newest=True,
links=(),
......@@ -131,6 +166,7 @@ class Eggs(object):
[develop_eggs_dir, eggs_dir]
)
else:
patch_dict = self._get_patch_dict(self.options)
ws = zc.buildout.easy_install.install(
distributions, eggs_dir,
links=links,
......@@ -138,9 +174,10 @@ class Eggs(object):
path=[develop_eggs_dir],
newest=newest,
allow_hosts=allow_hosts,
allow_unknown_extras=allow_unknown_extras)
allow_unknown_extras=allow_unknown_extras,
patch_dict=patch_dict)
ws = zc.buildout.easy_install.sort_working_set(
ws, eggs_dir, develop_eggs_dir
ws, buildout_dir, eggs_dir, develop_eggs_dir
)
cache_storage[cache_key] = ws
......
Patching eggs before installation
---------------------------------
The SlapOS extensions of ``zc.recipe.egg`` supports applying patches before installing eggs.
The syntax is to use a version with the magic string ``SlapOSPatched`` plus the number of
patches to apply.
Let's use a patch for demoneeded egg:
>>> write(sample_buildout, 'demoneeded.patch',
... r"""diff -ru before/demoneeded-1.1/eggrecipedemoneeded.py after/demoneeded-1.1/eggrecipedemoneeded.py
... --- before/demoneeded-1.1/eggrecipedemoneeded.py 2020-09-08 09:27:36.000000000 +0200
... +++ after/demoneeded-1.1/eggrecipedemoneeded.py 2020-09-08 09:46:16.482243822 +0200
... @@ -1,3 +1,3 @@
... -y=1
... +y="patched demoneeded"
... def f():
... pass
... \ No newline at end of file
... """)
First, we install demoneeded directly:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = demoneeded
...
... [demoneeded]
... recipe = zc.recipe.egg:eggs
... eggs = demoneeded
... find-links = %(server)s
... index = %(server)s/index
... demoneeded-patches =
... ./demoneeded.patch#4b8ad56711dd0d898a2b7957e9604079
... demoneeded-patch-options = -p2
...
... [versions]
... demoneeded = 1.1+SlapOSPatched001
... """ % dict(server=link_server))
When running buildout, we have a warning that a different version is installed, but that's not fatal.
>>> print_(system(buildout), end='')
Installing demoneeded.
patching file eggrecipedemoneeded.py
Installing demoneeded 1.1
Caused installation of a distribution:
demoneeded 1.1+slapospatched001
with a different version.
The installed egg has the slapospatched001 marker
>>> ls(sample_buildout, 'eggs')
d demoneeded-1.1+slapospatched001-pyN.N.egg
- setuptools-0.7-py2.3.egg
d zc.buildout-1.0-py2.3.egg
The code of the egg has been patched:
>>> import glob
>>> import os.path
>>> cat(glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded-1.1+slapospatched001*', 'eggrecipedemoneeded.py'))[0])
y="patched demoneeded"
def f():
pass
Reset the state and also remove the installed egg
>>> remove('.installed.cfg')
>>> rmdir(glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded-1.1+slapospatched001*'))[0])
In the previous example we applied patches to an egg installed directly, but
the same technique can be used to apply patches on eggs installed as dependencies.
In this example we install demo and apply a patch to demoneeded, which is a dependency to demo.
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = demo
...
... [demo]
... recipe = zc.recipe.egg
... eggs = demo
... find-links = %(server)s
... index = %(server)s/index
... demoneeded-patches =
... ./demoneeded.patch#4b8ad56711dd0d898a2b7957e9604079
... demoneeded-patch-options = -p2
...
... [versions]
... demoneeded = 1.1+SlapOSPatched001
... """ % dict(server=link_server))
When running buildout, we also have that warning that a different version is installed.
>>> print_(system(buildout), end='')
Installing demo.
Getting distribution for 'demo'.
Got demo 0.3.
patching file eggrecipedemoneeded.py
Installing demoneeded 1.1
Caused installation of a distribution:
demoneeded 1.1+slapospatched001
with a different version.
Generated script '/sample-buildout/bin/demo'.
The installed egg has the slapospatched001 marker
>>> ls(sample_buildout, 'eggs')
d demo-0.3-pyN.N.egg
d demoneeded-1.1+slapospatched001-pyN.N.egg
- setuptools-0.7-py2.3.egg
d zc.buildout-1.0-py2.3.egg
If we run the demo script we see the patch was applied:
>>> print_(system(join(sample_buildout, 'bin', 'demo')), end='')
3 patched demoneeded
......@@ -100,6 +100,26 @@ def test_suite():
zc.buildout.testing.not_found,
])
),
doctest.DocFileSuite(
'patches.rst',
setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.normalize_script,
zc.buildout.testing.normalize_egg_py,
zc.buildout.tests.normalize_bang,
zc.buildout.tests.normalize_S,
zc.buildout.testing.not_found,
zc.buildout.testing.easy_install_deprecated,
(re.compile(r'[d-] zc.buildout(-\S+)?[.]egg(-link)?'),
'zc.buildout.egg'),
(re.compile(r'[d-] setuptools-[^-]+-'), 'setuptools-X-'),
(re.compile(r'eggs\\\\demo'), 'eggs/demo'),
(re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
])
),
]
if not WINDOWS:
suites.append(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment