Commit cd27772e authored by Xavier Thompson's avatar Xavier Thompson

[feat] Reimplement the extends algorithm

The new algorithm avoids fetching the same extended file more than once
and correctly handles overriding values and += and -=:

The new algorithm starts as if there was a buildout file containing

```
[buildout]
extends =
  user/defaults.cfg # if it exists
  buildout.cfg # if it exists
  command_line_extends.cfg # if passed on the command line
```

The files are then fetched in depth-first-search postorder and fetching
child nodes in the order given by the extends directive, ignoring files
that have already been fetched.

The buildout dicts are then collected in order, and this linearisation
is then merged at the end, overriding the first configs collected with
the later ones. The first dict in the linearisation is not from a file,
but the dict of buildout's (hardcoded) defaults. This is equivalent to
acting as though every file that does not extend anything extends these
defaults.

The first time a file must be downloaded from a url, the linearisation
is merged with the configs already collected, and the resulting options
are then used to determine the download options for this download, and
every subsequent download.

This is a break with buildout's current logic for download options.

By analogy with classes in Python, we are computing a linearisation of
the class hierarchy to determine the method resolution order (MRO).
This algorithm is not the same as Python's MRO since Python 2.3 (C3).

It could be good to switch to a C3 linearisation like Python.
parent b1632152
...@@ -29,6 +29,11 @@ try: ...@@ -29,6 +29,11 @@ try:
except ImportError: except ImportError:
from UserDict import DictMixin from UserDict import DictMixin
try:
from urllib.parse import urljoin
except ImportError: # BBB Py2
from urlparse import urljoin
import zc.buildout.configparser import zc.buildout.configparser
import copy import copy
import datetime import datetime
...@@ -359,54 +364,24 @@ class Buildout(DictMixin): ...@@ -359,54 +364,24 @@ class Buildout(DictMixin):
data['buildout']['directory'] = SectionKey( data['buildout']['directory'] = SectionKey(
os.path.dirname(config_file), 'COMPUTED_VALUE') os.path.dirname(config_file), 'COMPUTED_VALUE')
cloptions = dict( result = {}
(section, dict((option, SectionKey(value, 'COMMAND_LINE_VALUE')) for section, option, value in cloptions:
for (_, option, value) in v)) result.setdefault(section, {})[option] = value
for (section, v) in itertools.groupby(sorted(cloptions),
lambda v: v[0])
)
override = copy.deepcopy(cloptions.get('buildout', {}))
# load user defaults, which override defaults options = result.setdefault('buildout', {})
extends = []
user_config = _get_user_config() user_config = _get_user_config()
if use_user_defaults and os.path.exists(user_config): if use_user_defaults and os.path.exists(user_config):
download_options = data['buildout'] extends.append(user_config)
user_defaults, _ = _open(
os.path.dirname(user_config),
user_config, [], download_options,
override, set(), {}
)
for_download_options = _update(data, user_defaults)
else:
user_defaults = {}
for_download_options = copy.deepcopy(data)
# load configuration files
if config_file: if config_file:
download_options = for_download_options['buildout'] extends.append(config_file)
cfg_data, _ = _open( clextends = options.get('extends')
os.path.dirname(config_file), if clextends:
config_file, [], download_options, extends.append(clextends)
override, set(), user_defaults options['extends'] = '\n'.join(extends)
)
data = _update(data, cfg_data)
# extends from command-line
if 'buildout' in cloptions:
cl_extends = cloptions['buildout'].pop('extends', None)
if cl_extends:
for extends in cl_extends.value.split():
download_options = for_download_options['buildout']
cfg_data, _ = _open(
os.path.dirname(extends),
os.path.basename(extends),
[], download_options,
override, set(), user_defaults
)
data = _update(data, cfg_data)
# apply command-line options data = _extends(data, result, os.getcwd(), 'COMMAND_LINE_VALUE')
data = _update(data, cloptions)
# Set up versions section, if necessary # Set up versions section, if necessary
if 'versions' not in data['buildout']: if 'versions' not in data['buildout']:
...@@ -1593,7 +1568,7 @@ class Options(DictMixin): ...@@ -1593,7 +1568,7 @@ class Options(DictMixin):
raise zc.buildout.UserError("No section named %r" % iname) raise zc.buildout.UserError("No section named %r" % iname)
result.update(self._do_extend_raw(iname, raw, doing)) result.update(self._do_extend_raw(iname, raw, doing))
result = _annotate_section(result, "") _annotate_section(result, "")
data = _annotate_section(copy.deepcopy(data), "") data = _annotate_section(copy.deepcopy(data), "")
_update_section(result, data) _update_section(result, data)
_unannotate_section(result) _unannotate_section(result)
...@@ -1885,101 +1860,107 @@ def _default_globals(): ...@@ -1885,101 +1860,107 @@ def _default_globals():
variable_template_split = re.compile('([$]{[^}]*})').split variable_template_split = re.compile('([$]{[^}]*})').split
def _open(
base, filename, seen, download_options,
override, downloaded, user_defaults
):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found. class _extends(object):
"""
download_options = _update_section(download_options, override) def __new__(cls, defaults, *args):
raw_download_options = _unannotate_section(download_options) self = super(_extends, cls).__new__(cls)
newest = bool_option(raw_download_options, 'newest', 'false') self.seen = set()
fallback = newest and not (filename in downloaded) self.processing = []
extends_cache = raw_download_options.get('extends-cache') self.extends = [defaults]
self._download_options = []
self.collect(*args)
return self.merge()
def merge(self):
result = {}
for d in self.extends:
_update(result, d)
return result
def __getattr__(self, attr):
if attr == 'download_options':
# Compute processed options
result_so_far = self.merge()
self.extends[:] = [result_so_far]
value = copy.deepcopy(result_so_far.get('buildout')) or {}
# Update with currently-being-processed options
for options in reversed(self._download_options):
_update_section(value, options)
value = _unannotate_section(value)
setattr(self, attr, value)
return value
return self.__getattribute__(attr)
def collect(self, result, base, filename):
options = result.get('buildout', {})
extends = options.pop('extends', '')
# Sanitize buildout options
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
_annotate(result, filename)
# Collect extends and unprocessed download options
self.processing.append(filename)
self._download_options.append(options)
for fextends in extends.split():
self.open(base, fextends)
self.extends.append(result)
del self.processing[-1], self._download_options[-1]
def open(self, base, filename):
# Determine file location
if _isurl(filename):
download = True
elif _isurl(base):
download = True
filename = urljoin(base + '/', filename)
else:
download = False
filename = os.path.realpath(
os.path.join(base, os.path.expanduser(filename)))
# Detect repetitions and loops
if filename in self.seen:
if filename in self.processing:
raise zc.buildout.UserError("circular extends: %s" % filename)
return
self.seen.add(filename)
# Fetch file
is_temp = False
try:
if download:
download_options = self.download_options
extends_cache = download_options.get('extends-cache')
if extends_cache and variable_template_split(extends_cache)[1::2]: if extends_cache and variable_template_split(extends_cache)[1::2]:
raise ValueError( raise ValueError(
"extends-cache '%s' may not contain ${section:variable} to expand." "extends-cache '%s' may not contain ${section:variable} to expand."
% extends_cache % extends_cache
) )
download = zc.buildout.download.Download( downloaded_filename, is_temp = zc.buildout.download.Download(
raw_download_options, cache=extends_cache, download_options, cache=extends_cache,
fallback=fallback, hash_name=True) fallback=bool_option(download_options, 'newest'),
is_temp = False hash_name=True)(filename)
downloaded_filename = None filename_for_logging = '%s (downloaded as %s)' % (
if _isurl(filename): filename, downloaded_filename)
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')]
elif _isurl(base):
if os.path.isabs(filename):
fp = open(filename)
base = os.path.dirname(filename)
else:
filename = base + '/' + filename
downloaded_filename, is_temp = download(filename)
fp = open(downloaded_filename)
base = filename[:filename.rfind('/')] base = filename[:filename.rfind('/')]
else: else:
filename = os.path.join(base, filename) downloaded_filename = filename_for_logging = filename
fp = open(filename)
base = os.path.dirname(filename) base = os.path.dirname(filename)
downloaded.add(filename)
if filename in seen:
if is_temp:
fp.close()
os.remove(downloaded_filename)
raise zc.buildout.UserError("Recursive file include", seen, filename)
root_config_file = not seen
seen.append(filename)
filename_for_logging = filename with open(downloaded_filename) as fp:
if downloaded_filename:
filename_for_logging = '%s (downloaded as %s)' % (
filename, downloaded_filename)
result = zc.buildout.configparser.parse( result = zc.buildout.configparser.parse(
fp, filename_for_logging, _default_globals) fp, filename_for_logging, _default_globals)
finally:
fp.close()
if is_temp: if is_temp:
os.remove(downloaded_filename) os.remove(downloaded_filename)
options = result.get('buildout', {}) return self.collect(result, base, filename)
extends = options.pop('extends', None)
if 'extended-by' in options:
raise zc.buildout.UserError(
'No-longer supported "extended-by" option found in %s.' %
filename)
result = _annotate(result, filename)
if root_config_file and 'buildout' in result:
download_options = _update_section(
download_options, result['buildout']
)
if extends:
extends = extends.split()
eresult, user_defaults = _open(
base, extends.pop(0), seen, download_options, override,
downloaded, user_defaults
)
for fname in extends:
next_extend, user_defaults = _open(
base, fname, seen, download_options, override,
downloaded, user_defaults
)
eresult = _update(eresult, next_extend)
result = _update(eresult, result)
else:
if user_defaults:
result = _update(user_defaults, result)
user_defaults = {}
seen.pop()
return result, user_defaults
ignore_directories = '.svn', 'CVS', '__pycache__', '.git' ignore_directories = '.svn', 'CVS', '__pycache__', '.git'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment