Fix most Deprecation and Resource warnings.

On Python 3, these made the build output very hard to read (e.g., https://travis-ci.org/buildout/buildout/jobs/394026829)

On Python 3.7, these could actually break the doctests.

Closing the files should get us closer to being able to pass the tests with PyPy.

Makefile

@@ -51,4 +51,4 @@ clean:
 	rm -rf $(BUILD_DIRS) $(PYTHON_BUILD_DIR)
 
 test:
-	$(HERE)/bin/test -1 -v
+	$(HERE)/bin/test -1 -vvv -c

setup.py

@@ -18,7 +18,8 @@ import os
 from setuptools import setup
 
 def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+    with open(os.path.join(os.path.dirname(__file__), *rnames)) as f:
+        return f.read()
 
 doc_intro = """
 

src/zc/buildout/bootstrap.txt

@@ -19,7 +19,9 @@ Make sure the bootstrap script actually works::
     ... [buildout]
     ... parts =
     ... ''')
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f:
+    ...     bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
     ...     'bootstrap.py')); print_('X') # doctest: +ELLIPSIS
@@ -52,7 +54,7 @@ By default it gets the latest version::
     >>> buildout_script = join(sample_buildout, 'bin', 'buildout')
     >>> if sys.platform.startswith('win'):
     ...     buildout_script += '-script.py'
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-22.0.0...egg',
@@ -83,7 +85,7 @@ Now let's try with ``2.0.0``, which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',
@@ -103,7 +105,7 @@ Now let's try with ``31.0.0``, which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-...egg',
@@ -121,7 +123,7 @@ which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',
@@ -153,7 +155,7 @@ specify the setuptools version, and to reuse the setuptools zipfile::
     >>> os.path.exists('setuptools-32.1.0.zip')
     True
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',

src/zc/buildout/bootstrap_cl_settings.test

@@ -20,7 +20,8 @@ Some people pass buildout settings to bootstrap.
     ... [buildout]
     ... parts =
     ... ''')
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f: bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable) +
     ...     ' bootstrap.py buildout:directory=' + top +
@@ -43,7 +44,8 @@ They might do it with init, but no worries:
     >>> os.chdir(top)
     >>> mkdir(top, 'buildout')
     >>> os.chdir(top)
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f: bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable) +
     ...     ' bootstrap.py buildout:directory=' + top +

src/zc/buildout/buildout.py

@@ -586,7 +586,8 @@ class Buildout(DictMixin):
             if dist.precedence == pkg_resources.DEVELOP_DIST:
                 dest = os.path.join(options['develop-eggs-directory'],
                                     name+'.egg-link')
-                open(dest, 'w').write(dist.location)
+                with open(dest, 'w') as f:
+                  f.write(dist.location)
                 entries.append(dist.location)
             else:
                 dest = os.path.join(options['eggs-directory'],
@@ -905,8 +906,8 @@ class Buildout(DictMixin):
                     setup = self._buildout_path(setup)
                     files = glob.glob(setup)
                     if not files:
-                        self._logger.warn("Couldn't develop %r (not found)",
-                                          setup)
+                        self._logger.warning("Couldn't develop %r (not found)",
+                                             setup)
                     else:
                         files.sort()
                     for setup in files:
@@ -1138,8 +1139,8 @@ class Buildout(DictMixin):
         if (realpath(os.path.abspath(sys.argv[0])) != should_run):
             self._logger.debug("Running %r.", realpath(sys.argv[0]))
             self._logger.debug("Local buildout is %r.", should_run)
-            self._logger.warn("Not upgrading because not running a local "
-                              "buildout command.")
+            self._logger.warning("Not upgrading because not running a local "
+                                 "buildout command.")
             return
 
         self._logger.info("Upgraded:\n  %s;\nrestarting.",
@@ -1548,7 +1549,7 @@ class Options(DictMixin):
 
     _template_split = re.compile('([$]{[^}]*})').split
     _simple = re.compile('[-a-zA-Z0-9 ._]+$').match
-    _valid = re.compile('\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
+    _valid = re.compile(r'\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
     def _sub(self, template, seen, last=True):
         value = self._template_split(template)
         subs = []
@@ -1652,7 +1653,7 @@ class Options(DictMixin):
                     elif os.path.isfile(p):
                         os.remove(p)
                     else:
-                        self.buildout._logger.warn("Couldn't clean up %r.", p)
+                        self.buildout._logger.warning("Couldn't clean up %r.", p)
                 raise
         finally:
             self._created = None
@@ -2037,9 +2038,9 @@ def _check_for_unused_options_in_section(buildout, section):
     unused = [option for option in sorted(options._raw)
               if option not in options._data]
     if unused:
-        buildout._logger.warn("Unused options for %s: %s."
-                              % (section, ' '.join(map(repr, unused)))
-                              )
+        buildout._logger.warning("Unused options for %s: %s."
+                                 % (section, ' '.join(map(repr, unused)))
+        )
 
 _usage = """\
 Usage: buildout [options] [assignments] [command [command arguments]]

src/zc/buildout/buildout.txt

@@ -3342,3 +3342,7 @@ We see that our extension is loaded and executed::
     ext ['buildout', 'versions']
     Develop: '/sample-bootstrapped/demo'
     unload ['buildout', 'versions']
+
+..
+
+    >>> stop_server(server_url)

src/zc/buildout/configparser.py

@@ -178,7 +178,7 @@ def parse(fp, fpname, exp_globals=dict):
                     tail = tail.replace(';', '#') if tail else ''
                     # un-escape literal # and ; . Do not use a
                     # string-escape decode
-                    expr = expression.replace(r'\x23','#').replace(r'x3b', ';')
+                    expr = expression.replace(r'\x23','#').replace(r'\x3b', ';')
                     # rebuild a valid Python expression wrapped in a list
                     expr = head + expr + tail
                     # lazily populate context only expression

src/zc/buildout/configparser.test

@@ -33,7 +33,7 @@ First, an example that illustrates a well-formed configuration::
   b    += 1
 
   [s3]; comment
-  x =           a b        
+  x =           a b
 
 .. -> text
 
@@ -91,9 +91,9 @@ otherwise empty section) is blank.  For example:"
      'versions': {}}
 
 
-Sections headers can contain an optional arbitrary Python expression. 
+Sections headers can contain an optional arbitrary Python expression.
 When the expression evaluates to false the whole section is skipped.
-Several sections can have the same name with different expressions, enabling 
+Several sections can have the same name with different expressions, enabling
 conditional exclusion of sections::
 
   [s1: 2 + 2 == 4] # this expression is true [therefore "this section" _will_ be NOT skipped
@@ -105,7 +105,7 @@ conditional exclusion of sections::
   [   s2 : 41 + 1 == 42  ]  # a comment: this expression is [true], so this section will be kept
   long = b
 
-  [s3:2 in map(lambda i:i*2, [i for i in range(10)])] ;# Complex expressions are [possible!];, though they should not be (abused:) 
+  [s3:2 in map(lambda i:i*2, [i for i in range(10)])] ;# Complex expressions are [possible!];, though they should not be (abused:)
   # this section will not be skipped
   long = c
 
@@ -120,11 +120,11 @@ conditional exclusion of sections::
 
 
 Title line optional trailing comments are separated by a hash '#' or semicolon
-';' character.  The expression is an arbitrary expression with one restriction: 
-it cannot contain a literal hash '#' or semicolon ';' character: these need to be 
+';' character.  The expression is an arbitrary expression with one restriction:
+it cannot contain a literal hash '#' or semicolon ';' character: these need to be
 string-escaped.
-The comment can contain arbitrary characters, including brackets that are also 
-used to mark the end of a section header and may be ambiguous to recognize in 
+The comment can contain arbitrary characters, including brackets that are also
+used to mark the end of a section header and may be ambiguous to recognize in
 some cases. For example, valid sections lines include::
 
   [ a ]
@@ -177,7 +177,7 @@ some cases. For example, valid sections lines include::
 
 
 
-A title line optional trailing comment be separated by a hash or semicolon 
+A title line optional trailing comment be separated by a hash or semicolon
 character. The following are valid semicolon-separated comments::
 
   [ a ]  ;semicolon comment are supported for lines without expressions ]
@@ -241,10 +241,10 @@ The following sections with hash comment separators are valid too::
      'e': {'e': '1'}}
 
 
-However, explicit semicolon and hash characters are invalid in expressions and 
+However, explicit semicolon and hash characters are invalid in expressions and
 must be escaped or this triggers an error. In the rare case where a hash '#' or
-semicolon ';' would be needed in an expression literal, you can use the 
-string-escaped representation of these characters:  use '\x23' for hash '#' and 
+semicolon ';' would be needed in an expression literal, you can use the
+string-escaped representation of these characters:  use '\x23' for hash '#' and
 '\x3b' for semicolon ';' to avoid evaluation errors.
 These expressions are valid and use escaped hash and semicolons in literals::
 
@@ -278,11 +278,11 @@ And using unescaped semicolon and hash characters in expressions triggers an err
     ... except zc.buildout.configparser.MissingSectionHeaderError: pass # success
 
 
-One of the typical usage of expression is to have buildout parts that are 
-operating system or platform-specific.  The configparser.parse function has an 
-optional exp_globals argument.  This is a callable returning a mapping of 
-objects made available to the evaluation context of the expression. Here we add 
-the platform and sys modules to the evaluation context, so we can access 
+One of the typical usage of expression is to have buildout parts that are
+operating system or platform-specific.  The configparser.parse function has an
+optional exp_globals argument.  This is a callable returning a mapping of
+objects made available to the evaluation context of the expression. Here we add
+the platform and sys modules to the evaluation context, so we can access
 platform and sys modules functions and objects in our expressions ::
 
   [s1: str(platform.python_version_tuple()[0]) in ('2', '3',)] # this expression is true, the major versions of python are either 2 or 3
@@ -306,8 +306,8 @@ platform and sys modules functions and objects in our expressions ::
     {'s1': {'a': '1'}, 's2': {'long': 'b'}}
 
 
-Some limited (but hopefully sane and sufficient) default modules and 
-pre-computed common expressions available to an expression when the parser in 
+Some limited (but hopefully sane and sufficient) default modules and
+pre-computed common expressions available to an expression when the parser in
 called by buildout::
 
 
@@ -316,20 +316,20 @@ called by buildout::
   a = 1
 
   # major and minor python versions, yes even python 3.5 and 3.6 are there , prospectively
-  # comment: this expression "is true" and not that long expression cannot span several lines 
-  [s2: any([python2, python3, python24 , python25 , python26 , python27 , python30 , python31 , python32 , python33 , python34 , python35 , python36]) ] 
+  # comment: this expression "is true" and not that long expression cannot span several lines
+  [s2: any([python2, python3, python24 , python25 , python26 , python27 , python30 , python31 , python32 , python33 , python34 , python35 , python36]) ]
   b = 1
 
   # common python interpreter types
   [s3:cpython or pypy or jython or ironpython]  # a comment: this expression is likely always true, so this section will be kept
-  c = 1 
+  c = 1
 
   # common operating systems
-  [s4:linux or windows or cygwin or macosx or solaris or posix or True]  
+  [s4:linux or windows or cygwin or macosx or solaris or posix or True]
   d = 1
 
   # common bitness and endianness
-  [s5:bits32 or bits64 or little_endian or big_endian]  
+  [s5:bits32 or bits64 or little_endian or big_endian]
   e = 1
 
 .. -> text

src/zc/buildout/download.py

@@ -23,10 +23,20 @@ try:
     from urllib.request import FancyURLopener, URLopener, urlretrieve
     from urllib.parse import urlparse
     from urllib import request
+    import warnings
 
     class PatchedURLopener(FancyURLopener):
         http_error_default = URLopener.http_error_default
 
+        def __init__(self, *args, **kwargs):
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                # This creates a DeprecationWarning on 3.6:
+                # PatchedURLopener style of invoking requests is deprecated.
+                # Use newer urlopen functions/methods.
+                # Said warning breaks our doctests.
+                super(PatchedURLopener, self).__init__(*args, **kwargs)
+
     request._urlopener = PatchedURLopener()  # Ook! Monkey patch!
 
 except ImportError:

src/zc/buildout/easy_install.py

@@ -70,7 +70,7 @@ default_index_url = os.environ.get(
 logger = logging.getLogger('zc.buildout.easy_install')
 
 url_match = re.compile('[a-z0-9+.-]+://').match
-is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search
+is_source_encoding_line = re.compile(r'coding[:=]\s*([-\w.]+)').search
 # Source encoding regex from http://www.python.org/dev/peps/pep-0263/
 
 is_win32 = sys.platform == 'win32'

src/zc/buildout/repeatable.txt

@@ -397,7 +397,7 @@ at the end.
 
 The versions file now contains the extra pin:
 
-    >>> print_(open('my_versions.cfg').read()) # doctest: +ELLIPSIS
+    >>> with open('my_versions.cfg') as f: print_(f.read()) # doctest: +ELLIPSIS
     <BLANKLINE>
     ...
     # Added by buildout at YYYY-MM-DD hh:mm:ss.dddddd
@@ -437,7 +437,7 @@ printing them to the console):
 
 The versions file contains the extra pin:
 
-    >>> print_(open('my_versions.cfg').read()) # doctest: +ELLIPSIS
+    >>> with open('my_versions.cfg') as f: print_(f.read()) # doctest: +ELLIPSIS
     <BLANKLINE>
     [versions]
     ...

src/zc/buildout/rmtree.py

@@ -40,7 +40,7 @@ def rmtree (path):
     Now create a file ...
 
     >>> foo = os.path.join (d, 'foo')
-    >>> _ = open (foo, 'w').write ('huhu')
+    >>> with open (foo, 'w') as f: _ = f.write ('huhu')
     >>> bar = os.path.join (d, 'bar')
     >>> os.symlink(bar, bar)
 
@@ -107,4 +107,3 @@ def test_suite():
 
 if "__main__" == __name__:
     doctest.testmod()
-

src/zc/buildout/testing.py

@@ -112,6 +112,19 @@ def system(command, input='', with_exit_code=False):
     # https://github.com/buildout/buildout/pull/311
     # http://bugs.python.org/issue19884
     env = dict(os.environ, TERM='dumb')
+
+    # Beginning in Python 3.4, 'U' mode to open() is deprecated.
+    # Python 3.7 changes the way deprecations are shown for main
+    # modules, and introduces $PYTHONDEVMODE which turns on warnigs in
+    # more places. If that's done, this leads many of our doctests to
+    # break; some code path through executing setup.py does this, but
+    # it's not in our code. Unfortunately, normalizing this printed
+    # line away doesn't work, it just produces a blank line. We resort
+    # to turning that warning off.
+    warnings = env.get('PYTHONWARNINGS', '')
+    env['PYTHONWARNINGS'] = "ignore:'U' mode is deprecated:DeprecationWarning::," + warnings
+
+
     p = subprocess.Popen(command,
                          shell=True,
                          stdin=subprocess.PIPE,
@@ -131,6 +144,7 @@ def system(command, input='', with_exit_code=False):
         # Use the with_exit_code=True parameter when you want to test the exit
         # code of the command you're running.
         output += 'EXIT CODE: %s' % p.wait()
+    p.wait()
     return output
 
 def get(url):
@@ -293,6 +307,7 @@ def buildoutSetUp(test):
         sdist = sdist,
         bdist_egg = bdist_egg,
         start_server = start_server,
+        stop_server = stop_server,
         buildout = os.path.join(sample, 'bin', 'buildout'),
         wait_until = wait_until,
         print_ = print_,
@@ -330,6 +345,7 @@ class Handler(BaseHTTPRequestHandler):
 
     def do_GET(self):
         if '__stop__' in self.path:
+            self.__server.server_close()
             raise SystemExit
 
         def k():
@@ -401,6 +417,7 @@ def _run(tree, port):
     server_address = ('localhost', port)
     httpd = Server(tree, server_address, Handler)
     httpd.serve_forever()
+    httpd.server_close()
 
 def get_port():
     for i in range(10):
@@ -513,10 +530,10 @@ if sys.version_info > (2, ):
         re.compile('(\n?)d  __pycache__\n'), '\\1')
 else:
     normalize___pycache__ = (
-        re.compile('(\n?)-  \S+\.pyc\n'), '\\1')
+        re.compile(r'(\n?)-  \S+\.pyc\n'), '\\1')
 
 normalize_egg_py = (
-    re.compile('-py\d[.]\d(-\S+)?.egg'),
+    re.compile(r'-py\d[.]\d(-\S+)?.egg'),
     '-pyN.N.egg',
     )
 

src/zc/buildout/windows.txt

@@ -28,7 +28,7 @@ can't delete.
     ...             os.makedirs (self.location)
     ...
     ...         name = os.path.join (self.location, 'readonly.txt')
-    ...         open (name, 'w').write ('this is a read only file')
+    ...         with open (name, 'w') as f: f.write ('this is a read only file')
     ...         os.chmod(name, 256)
     ...         return ()
     ...

tox.ini

@@ -7,7 +7,7 @@ commands =
     # buildout's dev.py wants python to not have setuptools
     pip uninstall -y zc.buildout setuptools pip
     python dev.py
-    {toxinidir}/bin/test -1 -v -c
+    {toxinidir}/bin/test -1 -vvv -c
 # since we're removing setuptools we can't possibly reuse the virtualenv
 recreate = true
 # if the user has ccache installed, PATH may contain /usr/lib/ccache that has a

zc.recipe.egg_/setup.py

@@ -20,7 +20,8 @@ import os
 from setuptools import setup, find_packages
 
 def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+    with open(os.path.join(os.path.dirname(__file__), *rnames)) as f:
+        return f.read()
 
 name = "zc.recipe.egg"
 setup(

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -154,7 +154,7 @@ class Scripts(Eggs):
             assert relative_paths == 'false'
 
     parse_entry_point = re.compile(
-        '([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
+        r'([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
         ).match
     def install(self):
         reqs, ws = self.working_set()

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -50,9 +50,9 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                zc.buildout.tests.normalize_S,
                zc.buildout.testing.not_found,
-               (re.compile('[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
+               (re.compile(r'[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
-               (re.compile('[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
+               (re.compile(r'[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
                (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
                ])
@@ -66,15 +66,15 @@ def test_suite():
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.not_found,
                (re.compile('__buildout_signature__ = '
-                           'sample-\S+\s+'
-                           'zc.recipe.egg-\S+\s+'
-                           'setuptools-\S+\s+'
-                           'zc.buildout-\S+\s*'
+                           r'sample-\S+\s+'
+                           r'zc.recipe.egg-\S+\s+'
+                           r'setuptools-\S+\s+'
+                           r'zc.buildout-\S+\s*'
                            ),
                 '__buildout_signature__ = sample- zc.recipe.egg-'),
-               (re.compile('find-links = http://localhost:\d+/'),
+               (re.compile(r'find-links = http://localhost:\d+/'),
                 'find-links = http://localhost:8080/'),
-               (re.compile('index = http://localhost:\d+/index'),
+               (re.compile(r'index = http://localhost:\d+/index'),
                 'index = http://localhost:8080/index'),
                ])
             ),
@@ -87,7 +87,7 @@ def test_suite():
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.not_found,
                 (re.compile("(d  ((ext)?demo(needed)?|other)"
-                            "-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
+                            r"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
                  '\\1V.V.egg'),
                 (re.compile('extdemo.c\n.+\\\\extdemo.exp\n'), ''),
                 (re.compile(