Add support to ignore sections conditionally to a Python expression.

Section titles can now have this form:
 [sectionname:Python expression] # optional comment

If the Python expression evals to False, the section will be ignored.

Expressions have some defaults to support common conditions such as:
 [sectionname: not windows] # ignore this section on windows

Section title lines in the traditional form are still supported of
course:
 [sectionname] # optional comment 
or
 [sectionname] ; optional comment 

src/zc/buildout/buildout.py

@@ -1400,6 +1400,82 @@ def _save_options(section, options, f):
     for option, value in items:
         _save_option(option, value, f)
 
+def _default_globals():
+    """Return a mapping of default and precomputed expressions.
+    These default expressions are convenience defaults available when eveluating
+    section headers expressions.
+    NB: this is wrapped in a function so that the computing of these expressions
+    is lazy and done only if needed (ie if there is at least one section with 
+    an expression) because the computing of some of these expressions can be 
+    expensive. 
+    """
+    # partially derived or inspired from its.py
+    # Copyright (c) 2012, Kenneth Reitz All rights reserved.
+    # Redistribution and use in source and binary forms, with or without modification,
+    # are permitted provided that the following conditions are met:
+    # Redistributions of source code must retain the above copyright notice, this list
+    # of conditions and the following disclaimer. Redistributions in binary form must
+    # reproduce the above copyright notice, this list of conditions and the following
+    # disclaimer in the documentation and/or other materials provided with the
+    # distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+    # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+    # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+    # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+    # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+    # OF SUCH DAMAGE.
+
+    # default available modules, explicitly re-imported locally here on purpose
+    import sys
+    import os
+    import platform
+    import re
+
+    globals_defs = {'sys': sys, 'os': os, 'platform': platform, 're': re,}
+
+    # major python versions as python2 and python3
+    vertu = platform.python_version_tuple()
+    globals_defs.update({'python2': vertu[0] == '2', 'python3': vertu[0] == '3'})
+
+    # minor python versions as python24, python25 ... python36
+    pyver = ('24', '25', '26', '27', '30', '31', '32', '33', '34', '35', '36')
+    for v in pyver:
+        globals_defs['python' + v] = ''.join(vertu[:2]) == v
+
+    # interpreter type
+    sysver = sys.version.lower()
+    pypy = 'pypy' in sysver
+    jython = 'java' in sysver
+    ironpython ='iron' in sysver
+    # assume CPython, if nothing else.
+    cpython = not any((pypy, jython, ironpython,))
+    globals_defs.update({'cpython': cpython,
+                         'pypy': pypy,
+                         'jython': jython,
+                         'ironpython': ironpython})
+    # operating system
+    sysplat = str(sys.platform).lower()
+    globals_defs.update({'linux': 'linux' in sysplat,
+                         'windows': 'win32' in sysplat,
+                         'cygwin': 'cygwin' in sysplat,
+                         'solaris': 'sunos' in sysplat,
+                         'macosx': 'darwin' in sysplat,
+                         'posix': 'posix' in os.name.lower()})
+       
+    #bits and endianness
+    import struct
+    void_ptr_size = struct.calcsize('P') * 8
+    globals_defs.update({'bits32': void_ptr_size == 32,
+                         'bits64': void_ptr_size == 64,
+                         'little_endian': sys.byteorder == 'little',
+                         'big_endian': sys.byteorder == 'big'})
+
+    return globals_defs
+
 def _open(base, filename, seen, dl_options, override, downloaded):
     """Open a configuration file and return the result as a dictionary,
 
@@ -1441,7 +1517,7 @@ def _open(base, filename, seen, dl_options, override, downloaded):
     root_config_file = not seen
     seen.append(filename)
 
-    result = zc.buildout.configparser.parse(fp, filename)
+    result = zc.buildout.configparser.parse(fp, filename, _default_globals)
     fp.close()
     if is_temp:
         os.remove(path)

src/zc/buildout/configparser.py

@@ -19,6 +19,9 @@
 
 import re
 import textwrap
+import logging
+
+logger = logging.getLogger('zc.buildout')
 
 class Error(Exception):
     """Base class for ConfigParser exceptions."""
@@ -71,25 +74,76 @@ class MissingSectionHeaderError(ParsingError):
         self.lineno = lineno
         self.line = line
 
-section_header = re.compile(
-    r'\[\s*(?P<header>[^\s[\]:{}]+)\s*]\s*([#;].*)?$').match
+# This regex captures either plain sections headers with optional trailing 
+# comment separated by a semicolon or a pound sign OR ...
+# new style section headers with an expression and optional trailing comment 
+# that then can be only separated by a pound sign.
+# This second case could require complex parsing as expressions and comments
+# can contain brackets and # signs that would need at least to balance brackets
+# A title line with an expression has the general form:
+#  [section_name: some Python expression] # some comment
+# This regex leverages the fact that the following is a valid Python expression:
+#  [some Python expression] # some comment
+# and that section headers are always delimited by [brackets] which are also 
+# the delimiters for Python [lists]
+# So instead of doing complex parsing to balance brackets, we capture just 
+# enough from a header line to collect then remove the section_name and colon 
+# expression separator keeping only a list-enclosed expression and optional
+# comments. Therefore the parsing and validation of this resulting Python 
+# expression can be entirely delegated to the built-in Python eval compiler. 
+# The result of the evaluated expression is the always returned wrapped in a 
+# list with a single item that contains the original expression
+section_header  = re.compile(
+    r'(?P<head>\[)'            # opening bracket [ starts a section title line
+    r'\s*'
+    r'(?P<name>[^\s[\]:{}]+)'  # section name
+    r'\s*'
+    r'('
+     r']'                      # closing bracket ] 
+     r'\s*'
+     r'([#;].*)?$'             # optional trailing comment marked by '#' or ';'
+    r'|'                       # OR
+     r':'                      # optional ':' separator for expression
+     r'\s*'
+     r'(?P<tail>.*'            # optional arbitrary Python expression
+     r']'                      # closing bracket ] 
+     r'\s*'
+     r'\#?.*)$'                # optional trailing comment marked by '#'
+    r')'
+    ).match
+
 option_start = re.compile(
     r'(?P<name>[^\s{}[\]=:]+\s*[-+]?)'
     r'='
     r'(?P<value>.*)$').match
+
 leading_blank_lines = re.compile(r"^(\s*\n)+")
 
-def parse(fp, fpname):
+def parse(fp, fpname, exp_globals=None):
     """Parse a sectioned setup file.
 
-    The sections in setup file contains a title line at the top,
+    The sections in setup files contain a title line at the top,
     indicated by a name in square brackets (`[]'), plus key/value
     options lines, indicated by `name: value' format lines.
     Continuations are represented by an embedded newline then
     leading whitespace.  Blank lines, lines beginning with a '#',
     and just about everything else are ignored.
+
+    The title line is in the form [name] followed an optional a trailing 
+    comment separated by a semicolon ';' or a pound `#' sign. 
+    
+    Optionally the title line can have the form [name:expression] where 
+    expression is an arbitrary Python expression. Sections with an expression 
+    that evaluates to False are ignored. In this form, the optional trailing 
+    comment can only be marked by a pound # sign (semi-colon ; is not valid)
+
+    exp_globals is a callable returning a mapping of defaults used as globals 
+    during the evaluation of a section conditional expression.
     """
     sections = {}
+    # the current section condition, possibly updated from a section expression
+    section_condition = True
+    context = None
     cursect = None                            # None, or a dictionary
     blockmode = None
     optname = None
@@ -106,6 +160,9 @@ def parse(fp, fpname):
             continue # comment
 
         if line[0].isspace() and cursect is not None and optname:
+            if not section_condition:
+                #skip section based on its expression condition
+                continue
             # continuation line
             if blockmode:
                 line = line.rstrip()
@@ -115,10 +172,32 @@ def parse(fp, fpname):
                     continue
             cursect[optname] = "%s\n%s" % (cursect[optname], line)
         else:
-            mo = section_header(line)
-            if mo:
-                # section header
-                sectname = mo.group('header')
+            header = section_header(line)
+            if header:
+                # reset to True when starting a new section
+                section_condition = True 
+                sectname = header.group('name')
+
+                head = header.group('head') # the starting [
+                tail = header.group('tail') # closing ], expression and comment
+                if tail:
+                    # lazily populate context only expression
+                    if not context:
+                        context = exp_globals() if exp_globals else {}
+
+                    # rebuild a valid Python expression wrapped in a list
+                    expression = head + tail
+
+                    # by design and construction, the evaluated  expression 
+                    # is always the first element of a wrapping list
+                    # so we get the first element 
+                    section_condition = eval(expression, context)[0]
+
+                    # ignore section when an expression evaluates to false
+                    if not section_condition:
+                        logger.debug('Ignoring section %(sectname)r with [expression]: %(expression)r' % locals())
+                        continue
+
                 if sectname in sections:
                     cursect = sections[sectname]
                 else:
@@ -133,6 +212,9 @@ def parse(fp, fpname):
             else:
                 mo = option_start(line)
                 if mo:
+                    if not section_condition:
+                        # filter out options of conditionally ignored section
+                        continue
                     # option start line
                     optname, optval = mo.group('name', 'value')
                     optname = optname.rstrip()

src/zc/buildout/configparser.test

@@ -89,3 +89,187 @@ otherwise empty section) is blank.  For example:"
                'on_update': 'true',
                'recipe': 'collective.recipe.cmd'},
      'versions': {}}
+
+
+Sections headers can contain an optional arbitrary Python expression. 
+When the expression evaluates to false the whole section is skipped.
+Several sections can have the same name with different expressions, enabling 
+conditional exclusion of sections::
+
+  [s1: 2 + 2 == 4] # this expression is true [therefore "this section" _will_ be NOT skipped
+  a = 1
+
+  [   s2 : 2 + 2 == 5  ]         # comment: this expression is false, so this section will be ignored
+  long = a
+
+  [   s2 : 41 + 1 == 42  ]  # a comment: this expression is true, so this section will be kept
+  long = b
+
+  [s3:2 in map(lambda i:i*2, [i for i in range(10)])] # Complex expressions are [possible!];, though they should not be (abused:) 
+  # this section will not be skipped
+  long = c
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'s1': {'a': '1'}, 's2': {'long': 'b'}, 's3': {'long': 'c'}}
+
+
+The title line can contain an optional trailing comment separated by a pound 
+sign. The expression and the comment can contain arbitrary characters, including
+brackets that are also used to mark the end of a section header and  that may be 
+ambiguous to recognize in some cases. For example, valid sections lines include::
+
+  [ a ]
+  a=1
+
+  [ b ]  # []
+  b=1
+
+  [ c : True ]  # ]
+  c =1
+
+  [ d :  True]  # []
+  d=1
+
+  [ e ]  # []
+  e = 1
+
+  [ f ]  # ]
+  f = 1
+
+  [g:2 in map(lambda i:i*2, ['''#;)'''] + [i for i in range(10)] + list('#[]][;#'))] # Complex #expressions; ][are [possible!]
+  g = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'},
+     'b': {'b': '1'},
+     'c': {'c': '1'},
+     'd': {'d': '1'},
+     'e': {'e': '1'},
+     'f': {'f': '1'},
+     'g': {'g': '1'}}
+
+
+A title line optional trailing comment may also be separated by a comma 
+-- for backward compatibility -- if and only if the title line does not contain
+an expression. The following are valid::
+
+  [ a ]  ;comma comment are supported for lines without expressions ]
+  a = 1
+
+  # this comma separated comment is valid because this section does not contain an expression
+  [ b ]  ; []
+  b = 1
+
+  # this comma separated comment is valid because this section does not contain an expression
+  [ c ]  ; ]
+  c = 1
+
+  # this comma separated comment is valid because this section does not contain an expression
+  [ d ]  ; [
+  d = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test'))
+    {'a': {'a': '1'}, 'b': {'b': '1'}, 'c': {'c': '1'}, 'd': {'d': '1'}}
+
+
+
+And the following is invalid and will trigger an error::
+
+  [ d: True ]  ;comma comment are not supported for lines with expressions ]
+  d = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import zc.buildout.configparser
+    >>> try: zc.buildout.configparser.parse(StringIO.StringIO(text), 'test')
+    ... except SyntaxError: pass # success
+
+
+One of the typical usage is to have buildout parts that are operating system or
+platform specific.  The configparser.parse function has an optional 
+exp_globals argument.  This is a callable returning a mapping of objects made 
+available to the evaluation context of the expression. Here we add the 
+platform and sys modules to the evaluation context, so we can access platform 
+and sys functions and objects in our expressions ::
+
+  [s1: platform.python_version_tuple()[0] in ('2', '3',)] # this expression is true, the major versions of python are either 2 or 3
+  a = 1
+
+  [s2:sys.version[0] == '0'] # comment: this expression "is false",  there no major version 0 of Python so this section will be ignored
+  long = a
+
+  [s2:len(platform.uname()) > 0]  # a comment: this expression is likely always true, so this section will be kept
+  long = b
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> import platform, sys
+    >>> globs = lambda: {'platform': platform, 'sys': sys}
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test', exp_globals=globs))
+    {'s1': {'a': '1'}, 's2': {'long': 'b'}}
+
+
+Some limited (but hopefully sane and sufficient) default modules and 
+pre-computed common expressions available to an expression when the parser in 
+called by buildout::
+
+
+  #imported modules
+  [s1: sys and re and os and platform] # this expression is true: these modules are available
+  a = 1
+
+  # major and minor python versions, yes even python 3.5 and 3.6 are there , prospectively
+  # comment: this expression "is true" and not that long expression cannot span several lines 
+  [s2: any([python2, python3, python24 , python25 , python26 , python27 , python30 , python31 , python32 , python33 , python34 , python35 , python36]) ] 
+  b = 1
+
+  # common python interpreter types
+  [s3:cpython or pypy or jython or ironpython]  # a comment: this expression is likely always true, so this section will be kept
+  c = 1 
+
+  # common operating systems
+  [s4:linux or windows or cygwin or macosx or solaris or posix or True]  
+  d = 1
+
+  # common bitness and endianness
+  [s5:bits32 or bits64 or little_endian or big_endian]  
+  e = 1
+
+.. -> text
+
+    >>> try: import StringIO
+    ... except ImportError: import io as StringIO
+    >>> import pprint, zc.buildout.configparser
+    >>> import zc.buildout.buildout
+    >>> pprint.pprint(zc.buildout.configparser.parse(StringIO.StringIO(
+    ...     text), 'test', zc.buildout.buildout._default_globals))
+    {'s1': {'a': '1'},
+     's2': {'b': '1'},
+     's3': {'c': '1'},
+     's4': {'d': '1'},
+     's5': {'e': '1'}}
+