Commit 2f348893 authored by Jim Fulton's avatar Jim Fulton

Merge pull request #76 from pombredanne/header_expressions

Add support to ignore sections conditionally based on a Python expression.
parents bbfa363c b549149c
...@@ -1400,6 +1400,85 @@ def _save_options(section, options, f): ...@@ -1400,6 +1400,85 @@ def _save_options(section, options, f):
for option, value in items: for option, value in items:
_save_option(option, value, f) _save_option(option, value, f)
def _default_globals():
"""Return a mapping of default and precomputed expressions.
These default expressions are convenience defaults available when eveluating
section headers expressions.
NB: this is wrapped in a function so that the computing of these expressions
is lazy and done only if needed (ie if there is at least one section with
an expression) because the computing of some of these expressions can be
expensive.
"""
# partially derived or inspired from its.py
# Copyright (c) 2012, Kenneth Reitz All rights reserved.
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer. Redistributions in binary form must
# reproduce the above copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided with the
# distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
# OF SUCH DAMAGE.
# default available modules, explicitly re-imported locally here on purpose
import sys
import os
import platform
import re
globals_defs = {'sys': sys, 'os': os, 'platform': platform, 're': re,}
# major python major_python_versions as python2 and python3
major_python_versions = platform.python_version_tuple()
globals_defs.update({'python2': major_python_versions[0] == '2',
'python3': major_python_versions[0] == '3'})
# minor python major_python_versions as python24, python25 ... python36
minor_python_versions = ('24', '25', '26', '27',
'30', '31', '32', '33', '34', '35', '36')
for v in minor_python_versions:
globals_defs['python' + v] = ''.join(major_python_versions[:2]) == v
# interpreter type
sys_version = sys.version.lower()
pypy = 'pypy' in sys_version
jython = 'java' in sys_version
ironpython ='iron' in sys_version
# assume CPython, if nothing else.
cpython = not any((pypy, jython, ironpython,))
globals_defs.update({'cpython': cpython,
'pypy': pypy,
'jython': jython,
'ironpython': ironpython})
# operating system
sys_platform = str(sys.platform).lower()
globals_defs.update({'linux': 'linux' in sys_platform,
'windows': 'win32' in sys_platform,
'cygwin': 'cygwin' in sys_platform,
'solaris': 'sunos' in sys_platform,
'macosx': 'darwin' in sys_platform,
'posix': 'posix' in os.name.lower()})
#bits and endianness
import struct
void_ptr_size = struct.calcsize('P') * 8
globals_defs.update({'bits32': void_ptr_size == 32,
'bits64': void_ptr_size == 64,
'little_endian': sys.byteorder == 'little',
'big_endian': sys.byteorder == 'big'})
return globals_defs
def _open(base, filename, seen, dl_options, override, downloaded): def _open(base, filename, seen, dl_options, override, downloaded):
"""Open a configuration file and return the result as a dictionary, """Open a configuration file and return the result as a dictionary,
...@@ -1441,7 +1520,7 @@ def _open(base, filename, seen, dl_options, override, downloaded): ...@@ -1441,7 +1520,7 @@ def _open(base, filename, seen, dl_options, override, downloaded):
root_config_file = not seen root_config_file = not seen
seen.append(filename) seen.append(filename)
result = zc.buildout.configparser.parse(fp, filename) result = zc.buildout.configparser.parse(fp, filename, _default_globals)
fp.close() fp.close()
if is_temp: if is_temp:
os.remove(path) os.remove(path)
......
...@@ -19,6 +19,9 @@ ...@@ -19,6 +19,9 @@
import re import re
import textwrap import textwrap
import logging
logger = logging.getLogger('zc.buildout')
class Error(Exception): class Error(Exception):
"""Base class for ConfigParser exceptions.""" """Base class for ConfigParser exceptions."""
...@@ -71,25 +74,68 @@ class MissingSectionHeaderError(ParsingError): ...@@ -71,25 +74,68 @@ class MissingSectionHeaderError(ParsingError):
self.lineno = lineno self.lineno = lineno
self.line = line self.line = line
# This regex captures either sections headers with optional trailing comment
# separated by a semicolon or a hash. Section headers can have an optional
# expression. Expressions and comments can contain brackets but no verbatim '#'
# and ';' : these need to be escaped.
# A title line with an expression has the general form:
# [section_name: some Python expression] #; some comment
# This regex leverages the fact that the following is a valid Python expression:
# [some Python expression] # some comment
# and that section headers are also delimited by [brackets] taht are also [list]
# delimiters.
# So instead of doing complex parsing to balance brackets in an expression, we
# capture just enough from a header line to collect then remove the section_name
# and colon expression separator keeping only a list-enclosed expression and
# optional comments. The parsing and validation of this Python expression can be
# entirely delegated to Python's eval. The result of the evaluated expression is
# the always returned wrapped in a list with a single item that contains the
# original expression
section_header = re.compile( section_header = re.compile(
r'\[\s*(?P<header>[^\s[\]:{}]+)\s*]\s*([#;].*)?$').match r'(?P<head>\[)'
r'\s*'
r'(?P<name>[^\s#[\]:;{}]+)'
r'\s*'
r'(:(?P<expression>[^#;]*))?'
r'\s*'
r'(?P<tail>]'
r'\s*'
r'([#;].*)?$)'
).match
option_start = re.compile( option_start = re.compile(
r'(?P<name>[^\s{}[\]=:]+\s*[-+]?)' r'(?P<name>[^\s{}[\]=:]+\s*[-+]?)'
r'=' r'='
r'(?P<value>.*)$').match r'(?P<value>.*)$').match
leading_blank_lines = re.compile(r"^(\s*\n)+") leading_blank_lines = re.compile(r"^(\s*\n)+")
def parse(fp, fpname): def parse(fp, fpname, exp_globals=None):
"""Parse a sectioned setup file. """Parse a sectioned setup file.
The sections in setup file contains a title line at the top, The sections in setup files contain a title line at the top,
indicated by a name in square brackets (`[]'), plus key/value indicated by a name in square brackets (`[]'), plus key/value
options lines, indicated by `name: value' format lines. options lines, indicated by `name: value' format lines.
Continuations are represented by an embedded newline then Continuations are represented by an embedded newline then
leading whitespace. Blank lines, lines beginning with a '#', leading whitespace. Blank lines, lines beginning with a '#',
and just about everything else are ignored. and just about everything else are ignored.
The title line is in the form [name] followed by an optional trailing
comment separated by a semicolon `;' or a hash `#' character.
Optionally the title line can have the form `[name:expression]' where
expression is an arbitrary Python expression. Sections with an expression
that evaluates to False are ignored. Semicolon `;' an hash `#' characters
mustr be string-escaped in expression literals.
exp_globals is a callable returning a mapping of defaults used as globals
during the evaluation of a section conditional expression.
""" """
sections = {} sections = {}
# the current section condition, possibly updated from a section expression
section_condition = True
context = None
cursect = None # None, or a dictionary cursect = None # None, or a dictionary
blockmode = None blockmode = None
optname = None optname = None
...@@ -106,6 +152,9 @@ def parse(fp, fpname): ...@@ -106,6 +152,9 @@ def parse(fp, fpname):
continue # comment continue # comment
if line[0].isspace() and cursect is not None and optname: if line[0].isspace() and cursect is not None and optname:
if not section_condition:
#skip section based on its expression condition
continue
# continuation line # continuation line
if blockmode: if blockmode:
line = line.rstrip() line = line.rstrip()
...@@ -115,10 +164,32 @@ def parse(fp, fpname): ...@@ -115,10 +164,32 @@ def parse(fp, fpname):
continue continue
cursect[optname] = "%s\n%s" % (cursect[optname], line) cursect[optname] = "%s\n%s" % (cursect[optname], line)
else: else:
mo = section_header(line) header = section_header(line)
if mo: if header:
# section header # reset to True when starting a new section
sectname = mo.group('header') section_condition = True
sectname = header.group('name')
head = header.group('head') # the starting [
expression = header.group('expression')
tail = header.group('tail') # closing ]and comment
if expression:
# normalize tail comments to Python style
tail = tail.replace(';', '#') if tail else ''
# un-escape literal # and ; . Do not use a string-escape decode
expr = expression.replace(r'\x23','#').replace(r'x3b', ';')
# rebuild a valid Python expression wrapped in a list
expr = head + expr + tail
# lazily populate context only expression
if not context:
context = exp_globals() if exp_globals else {}
# evaluated expression is in list: get first element
section_condition = eval(expr, context)[0]
# finally, ignore section when an expression evaluates to false
if not section_condition:
logger.debug('Ignoring section %(sectname)r with [expression]: %(expression)r' % locals())
continue
if sectname in sections: if sectname in sections:
cursect = sections[sectname] cursect = sections[sectname]
else: else:
...@@ -133,6 +204,9 @@ def parse(fp, fpname): ...@@ -133,6 +204,9 @@ def parse(fp, fpname):
else: else:
mo = option_start(line) mo = option_start(line)
if mo: if mo:
if not section_condition:
# filter out options of conditionally ignored section
continue
# option start line # option start line
optname, optval = mo.group('name', 'value') optname, optval = mo.group('name', 'value')
optname = optname.rstrip() optname = optname.rstrip()
......
...@@ -89,3 +89,260 @@ otherwise empty section) is blank. For example:" ...@@ -89,3 +89,260 @@ otherwise empty section) is blank. For example:"
'on_update': 'true', 'on_update': 'true',
'recipe': 'collective.recipe.cmd'}, 'recipe': 'collective.recipe.cmd'},
'versions': {}} 'versions': {}}
Sections headers can contain an optional arbitrary Python expression.
When the expression evaluates to false the whole section is skipped.
Several sections can have the same name with different expressions, enabling
conditional exclusion of sections::
[s1: 2 + 2 == 4] # this expression is true [therefore "this section" _will_ be NOT skipped
a = 1
[ s2 : 2 + 2 == 5 ] # comment: this expression is false, so this section will be ignored]
long = a
[ s2 : 41 + 1 == 42 ] # a comment: this expression is [true], so this section will be kept
long = b
[s3:2 in map(lambda i:i*2, [i for i in range(10)])] ;# Complex expressions are [possible!];, though they should not be (abused:)
# this section will not be skipped
long = c
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test'))
{'s1': {'a': '1'}, 's2': {'long': 'b'}, 's3': {'long': 'c'}}
Title line optional trailing comments are separated by a hash '#' or semicolon
';' character. The expression is an arbitrary expression with one restriction:
it cannot contain a literal hash '#' or semicolon ';' character: these need to be
string-escaped.
The comment can contain arbitrary characters, including brackets that are also
used to mark the end of a section header and may be ambiguous to recognize in
some cases. For example, valid sections lines include::
[ a ]
a=1
[ b ] # []
b=1
[ c : True ] # ]
c =1
[ d : True] # []
d=1
[ e ] # []
e = 1
[ f ] # ]
f = 1
[g:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] # Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
g = 1
[ h : True ] ; ]
h =1
[ i : True] ; []
i=1
[j:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] ; Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
j = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test'))
{'a': {'a': '1'},
'b': {'b': '1'},
'c': {'c': '1'},
'd': {'d': '1'},
'e': {'e': '1'},
'f': {'f': '1'},
'g': {'g': '1'},
'h': {'h': '1'},
'i': {'i': '1'},
'j': {'j': '1'}}
A title line optional trailing comment be separated by a hash or semicolon
character. The following are valid semicolon-separated comments::
[ a ] ;semicolon comment are supported for lines without expressions ]
a = 1
[ b ] ; []
b = 1
[ c ] ; ]
c = 1
[ d ] ; [
d = 1
[ e: True ] ;semicolon comments are supported for lines with expressions ]
e = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test'))
{'a': {'a': '1'},
'b': {'b': '1'},
'c': {'c': '1'},
'd': {'d': '1'},
'e': {'e': '1'}}
The following sections with hash comment separators are valid too::
[ a ] #hash comment ] are supported for lines without expressions ]
a = 1
[ b ] # []
b = 1
[ c ] # ]
c = 1
[ d ] # [
d = 1
[ e: True ] #hash comments] are supported for lines with expressions ]
e = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test'))
{'a': {'a': '1'},
'b': {'b': '1'},
'c': {'c': '1'},
'd': {'d': '1'},
'e': {'e': '1'}}
However, explicit semicolon and hash characters are invalid in expressions and
must be escaped or this triggers an error. In the rare case where a hash '#' or
semicolon ';' would be needed in an expression literal, you can use the
string-escaped representation of these characters: use '\x23' for hash '#' and
'\x3b' for semicolon ';' to avoid evaluation errors.
These expressions are valid and use escaped hash and semicolons in literals::
[a:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] # Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
a = 1
[b:2 in map(lambda i:i*2, ['''\x23\x3b)'''] + [i for i in range(10)] + list('\x23[]][\x3b\x23'))] ; Complex #expressions; ][are [possible!] and can us escaped # and ; in literals
b = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test'))
{'a': {'a': '1'}, 'b': {'b': '1'}}
And using unescaped semicolon and hash characters in expressions triggers an error::
[a:'#' in '#;'] # this is not a supported expression
a = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import zc.buildout.configparser
>>> try: zc.buildout.configparser.parse(StringIO.StringIO(text), 'test')
... except zc.buildout.configparser.MissingSectionHeaderError: pass # success
One of the typical usage of expression is to have buildout parts that are
operating system or platform-specific. The configparser.parse function has an
optional exp_globals argument. This is a callable returning a mapping of
objects made available to the evaluation context of the expression. Here we add
the platform and sys modules to the evaluation context, so we can access
platform and sys modules functions and objects in our expressions ::
[s1: platform.python_version_tuple()[0] in ('2', '3',)] # this expression is true, the major versions of python are either 2 or 3
a = 1
[s2:sys.version[0] == '0'] # comment: this expression "is false", there no major version 0 of Python so this section will be ignored
long = a
[s2:len(platform.uname()) > 0] # a comment: this expression is likely always true, so this section will be kept
long = b
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> import platform, sys
>>> globs = lambda: {'platform': platform, 'sys': sys}
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test', exp_globals=globs))
{'s1': {'a': '1'}, 's2': {'long': 'b'}}
Some limited (but hopefully sane and sufficient) default modules and
pre-computed common expressions available to an expression when the parser in
called by buildout::
#imported modules
[s1: sys and re and os and platform] # this expression is true: these modules are available
a = 1
# major and minor python versions, yes even python 3.5 and 3.6 are there , prospectively
# comment: this expression "is true" and not that long expression cannot span several lines
[s2: any([python2, python3, python24 , python25 , python26 , python27 , python30 , python31 , python32 , python33 , python34 , python35 , python36]) ]
b = 1
# common python interpreter types
[s3:cpython or pypy or jython or ironpython] # a comment: this expression is likely always true, so this section will be kept
c = 1
# common operating systems
[s4:linux or windows or cygwin or macosx or solaris or posix or True]
d = 1
# common bitness and endianness
[s5:bits32 or bits64 or little_endian or big_endian]
e = 1
.. -> text
>>> try: import StringIO
... except ImportError: import io as StringIO
>>> import pprint, zc.buildout.configparser
>>> import zc.buildout.buildout
>>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
... text), 'test', zc.buildout.buildout._default_globals))
{'s1': {'a': '1'},
's2': {'b': '1'},
's3': {'c': '1'},
's4': {'d': '1'},
's5': {'e': '1'}}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment