Update to zope.testbrowser 5.0 and its WebTest based implementation.

This removes the mechanize dependency.

CHANGES.rst

@@ -20,12 +20,15 @@ Features Added
 
 - Updated distributions:
 
+    - zope.testbrowser = 5.0
     - zope.globalrequest = 1.3
     - zope.testing = 4.6.0
 
 Restructuring
 +++++++++++++
 
+- Update to zope.testbrowser 5.0 and its WebTest based implementation.
+
 - Use `@implementer` and `@adapter` class decorators.
 
 4.0a2 (2016-09-09)

buildout.cfg

@@ -19,6 +19,7 @@ parts =
     requirements
 sources-dir = develop
 auto-checkout =
+    ZServer
 
 
 [test]

requirements-full.txt

@@ -26,7 +26,6 @@ ZODB==5.0.0
 ZServer==4.0a1
 five.globalrequest==1.0
 funcsigs==1.0.2
-mechanize==0.2.5
 mock==2.0.0
 pbr==1.10.0
 persistent==4.2.1
@@ -80,7 +79,7 @@ zope.size==4.1.0
 zope.structuredtext==4.1.0
 zope.tal==4.2.0
 zope.tales==4.1.1
-zope.testbrowser==4.0.4
+zope.testbrowser==5.0.0
 zope.testing==4.6.0
 zope.testrunner==4.5.1
 zope.traversing==4.1.0

src/Testing/testbrowser.py

@@ -12,148 +12,54 @@
 #
 ##############################################################################
 """Support for using zope.testbrowser from Zope2.
-
-Mostly just copy and paste from zope.testbrowser.testing.
 """
 
-import io
-
-import mechanize
-from six.moves.urllib.request import HTTPHandler
-from zExceptions import status_reasons
+import transaction
 from zope.testbrowser import browser
 
-from Testing.ZopeTestCase.zopedoctest import functional
-
-try:
-    from http.client import HTTPMessage
-    from urllib.request import AbstractHTTPHandler
-except ImportError:
-    from httplib import HTTPMessage
-    from urllib2 import AbstractHTTPHandler
-
-
-class PublisherConnection(object):
-
-    def __init__(self, host, timeout=None):
-        self.caller = functional.http
-        self.host = host
-        self.response = None
-
-    def set_debuglevel(self, level):
-        pass
-
-    def _quote(self, url):
-        # the publisher expects to be able to split on whitespace, so we have
-        # to make sure there is none in the URL
-        return url.replace(' ', '%20')
-
-    def request(self, method, url, body=None, headers=None):
-        """Send a request to the publisher.
-
-        The response will be stored in ``self.response``.
-        """
-        if body is None:
-            body = ''
-
-        if url == '':
-            url = '/'
+from Testing.ZopeTestCase.functional import savestate
+from Testing.ZopeTestCase.sandbox import AppZapper
+from Testing.ZopeTestCase.zopedoctest.functional import auth_header
+from ZPublisher.httpexceptions import HTTPExceptionHandler
+from ZPublisher.WSGIPublisher import publish_module
 
-        url = self._quote(url)
-        # Extract the handle_error option header
-        handle_errors_key = 'X-Zope-Handle-Errors'
-        handle_errors_header = headers.get(handle_errors_key, True)
-        if handle_errors_key in headers:
-            del headers[handle_errors_key]
-        # Translate string to boolean.
-        handle_errors = {'False': False}.get(handle_errors_header, True)
 
-        # Construct the headers.
-        header_chunks = []
-        if headers is not None:
-            for header in headers.items():
-                header_chunks.append('%s: %s' % header)
-            headers = '\n'.join(header_chunks) + '\n'
-        else:
-            headers = ''
+class WSGITestApp(object):
 
-        # Construct the full HTTP request string, since that is what the
-        # ``HTTPCaller`` wants.
-        request_string = (method + ' ' + url + ' HTTP/1.1\n' +
-                          headers + '\n' + body)
-        self.response = self.caller(request_string, handle_errors)
+    def __init__(self, browser):
+        self.browser = browser
 
-    def getresponse(self):
-        """Return a ``urllib`` compatible response.
+    @savestate
+    def __call__(self, environ, start_response):
+        # This is similar to
+        # Testing.ZopeTestCase.zopedoctest.functional.http
 
-        The goal of ths method is to convert the Zope Publisher's response to
-        a ``urllib`` compatible response, which is also understood by
-        mechanize.
-        """
-        real_response = self.response._response
-        status = real_response.getStatus()
-        reason = status_reasons[real_response.status]
+        # Commit previously done work
+        transaction.commit()
 
-        # Replace HTTP/1.1 200 OK with Status: 200 OK line.
-        headers = ['Status: %s %s' % (status, reason)]
-        wsgi_headers = self.response._wsgi_headers.getvalue().split('\r\n')
-        headers += [line for line in wsgi_headers[1:]]
-        headers = '\r\n'.join(headers)
-        content = self.response.getBody()
-        return PublisherResponse(content, headers, status, reason)
+        # Base64 encode auth header
+        http_auth = 'HTTP_AUTHORIZATION'
+        if http_auth in environ:
+            environ[http_auth] = auth_header(environ[http_auth])
 
+        publish = publish_module
+        if self.browser.handleErrors:
+            publish = HTTPExceptionHandler(publish)
+        wsgi_result = publish(environ, start_response)
 
-class PublisherResponse(object):
-    """``mechanize`` compatible response object."""
+        # Sync transaction
+        AppZapper().app()._p_jar.sync()
 
-    def __init__(self, content, headers, status, reason):
-        self.content = content
-        self.status = status
-        self.reason = reason
-        self.msg = HTTPMessage(io.BytesIO(headers), 0)
-        self.content_as_file = io.BytesIO(self.content)
-
-    def read(self, amt=None):
-        return self.content_as_file.read(amt)
-
-    def close(self):
-        """To overcome changes in mechanize and socket in python2.5"""
-        pass
-
-
-class PublisherHTTPHandler(HTTPHandler):
-    """Special HTTP handler to use the Zope Publisher."""
-
-    http_request = AbstractHTTPHandler.do_request_
-
-    def http_open(self, req):
-        """Open an HTTP connection having a ``urllib`` request."""
-        # Here we connect to the publisher.
-        return self.do_open(PublisherConnection, req)
-
-
-class PublisherMechanizeBrowser(mechanize.Browser):
-    """Special ``mechanize`` browser using the Zope Publisher HTTP handler."""
-
-    default_schemes = ['http']
-    default_others = ['_http_error', '_http_request_upgrade',
-                      '_http_default_error']
-    default_features = ['_redirect', '_cookies', '_referer', '_refresh',
-                        '_equiv', '_basicauth', '_digestauth']
-
-    def __init__(self, *args, **kws):
-        self.handler_classes = mechanize.Browser.handler_classes.copy()
-        self.handler_classes["http"] = PublisherHTTPHandler
-        self.default_others = [cls for cls in self.default_others
-                               if cls in mechanize.Browser.handler_classes]
-        mechanize.Browser.__init__(self, *args, **kws)
+        return wsgi_result
 
 
 class Browser(browser.Browser):
     """A Zope ``testbrowser` Browser that uses the Zope Publisher."""
 
-    def __init__(self, url=None):
-        mech_browser = PublisherMechanizeBrowser()
-        # override the http handler class
-        mech_browser.handler_classes["http"] = PublisherHTTPHandler
-        super(Browser, self).__init__(url=url, mech_browser=mech_browser)
+    handleErrors = True
+    raiseHttpErrors = True
+
+    def __init__(self, url=None, wsgi_app=None):
+        if wsgi_app is None:
+            wsgi_app = WSGITestApp(self)
+        super(Browser, self).__init__(url=url, wsgi_app=wsgi_app)

src/Testing/tests/test_testbrowser.py

@@ -11,12 +11,20 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-"""Unit tests for the testbrowser module.
+"""Tests for the testbrowser module.
 """
 
-import unittest
-from Testing.ZopeTestCase import FunctionalDocTestSuite
+from AccessControl.Permissions import view
+from six.moves.urllib.error import HTTPError
+from zExceptions import NotFound
+
 from OFS.SimpleItem import Item
+from Testing.testbrowser import Browser
+from Testing.ZopeTestCase import (
+    FunctionalTestCase,
+    user_name,
+    user_password,
+)
 
 
 class CookieStub(Item):
@@ -27,57 +35,102 @@ class CookieStub(Item):
         return 'Stub'
 
 
-def doctest_cookies():
-    """
-    We want to make sure that our testbrowser correctly understands
-    cookies.  We'll add a stub to ``self.folder`` that sets a cookie.
-
-        >>> from Testing.tests.test_testbrowser import CookieStub
-        >>> self.folder._setObject('stub', CookieStub())
-        'stub'
-
-    This response looks alright:
-
-        >>> response = self.publish('/test_folder_1_/stub')
-        >>> print(str(response)) #doctest: +ELLIPSIS
-        HTTP/1.1 200 OK
-        ...
-        Set-Cookie: evil="cookie"
-        ...
-
-    Let's try to look at the same folder with testbrowser:
-
-        >>> from Testing.testbrowser import Browser
-        >>> browser = Browser()
-        >>> browser.open('http://localhost/test_folder_1_/stub')
-        >>> 'Set-Cookie: evil="cookie"' in str(browser.headers)
-        True
-    """
+class ExceptionStub(Item):
+    """This is a stub, raising an exception."""
 
-
-def doctest_camel_case_headers():
-    """Make sure that the headers come out in camel case.
-
-    Some setup:
-
-        >>> from Testing.tests.test_testbrowser import CookieStub
-        >>> self.folder._setObject('stub', CookieStub())
-        'stub'
-
-    The Zope2 response mungs headers so they come out in camel case we should
-    do the same. We will test a few:
-
-        >>> from Testing.testbrowser import Browser
-        >>> browser = Browser()
-        >>> browser.open('http://localhost/test_folder_1_/stub')
-        >>> 'Content-Length: ' in str(browser.headers)
-        True
-        >>> 'Content-Type: ' in str(browser.headers)
-        True
-    """
+    def __call__(self, REQUEST):
+        raise ValueError('dummy')
+
+
+class TestTestbrowser(FunctionalTestCase):
+
+    def test_auth(self):
+        # Based on Testing.ZopeTestCase.testFunctional
+        basic_auth = '%s:%s' % (user_name, user_password)
+        self.folder.addDTMLDocument('secret_html', file='secret')
+        self.folder.secret_html.manage_permission(view, ['Owner'])
+        path = '/' + self.folder.absolute_url(1) + '/secret_html'
+
+        # Test direct publishing
+        response = self.publish(path + '/secret_html')
+        self.assertEqual(response.getStatus(), 401)
+        response = self.publish(path + '/secret_html', basic_auth)
+        self.assertEqual(response.getStatus(), 200)
+        self.assertEqual(response.getBody(), 'secret')
+
+        # Test browser
+        url = 'http://localhost' + path
+        browser = Browser()
+        browser.raiseHttpErrors = False
+        browser.open(url)
+        self.assertTrue(browser.headers['status'].startswith('401'))
+
+        browser.addHeader('Authorization', 'Basic ' + basic_auth)
+        browser.open(url)
+        self.assertTrue(browser.headers['status'].startswith('200'))
+        self.assertEqual(browser.contents, 'secret')
+
+    def test_cookies(self):
+        # We want to make sure that our testbrowser correctly
+        # understands cookies.
+        self.folder._setObject('stub', CookieStub())
+
+        # Test direct publishing
+        response = self.publish('/test_folder_1_/stub')
+        self.assertEqual(response.getCookie('evil')['value'], 'cookie')
+
+        browser = Browser()
+        browser.open('http://localhost/test_folder_1_/stub')
+        self.assertEqual(browser.cookies.get('evil'), '"cookie"')
+
+    def test_handle_errors_true(self):
+        self.folder._setObject('stub', ExceptionStub())
+        browser = Browser()
+        with self.assertRaises(HTTPError):
+            browser.open('http://localhost/test_folder_1_/stub')
+        self.assertTrue(browser.headers['status'].startswith('500'))
+
+        with self.assertRaises(HTTPError):
+            browser.open('http://localhost/nothing-is-here')
+        self.assertTrue(browser.headers['status'].startswith('404'))
+
+    def test_handle_errors_false(self):
+        self.folder._setObject('stub', ExceptionStub())
+        browser = Browser()
+        browser.handleErrors = False
+        with self.assertRaises(ValueError):
+            browser.open('http://localhost/test_folder_1_/stub')
+        self.assertTrue(browser.contents is None)
+
+        with self.assertRaises(NotFound):
+            browser.open('http://localhost/nothing-is-here')
+        self.assertTrue(browser.contents is None)
+
+    def test_raise_http_errors_false(self):
+        self.folder._setObject('stub', ExceptionStub())
+        browser = Browser()
+        browser.raiseHttpErrors = False
+
+        browser.open('http://localhost/test_folder_1_/stub')
+        self.assertTrue(browser.headers['status'].startswith('500'))
+
+        browser.open('http://localhost/nothing-is-here')
+        self.assertTrue(browser.headers['status'].startswith('404'))
+
+    def test_headers_camel_case(self):
+        # The Zope2 response mungs headers so they come out
+        # in camel case. We should do the same.
+        self.folder._setObject('stub', CookieStub())
+
+        browser = Browser()
+        browser.open('http://localhost/test_folder_1_/stub')
+        header_text = str(browser.headers)
+        self.assertTrue('Content-Length: ' in header_text)
+        self.assertTrue('Content-Type: ' in header_text)
 
 
 def test_suite():
-    return unittest.TestSuite((
-        FunctionalDocTestSuite(),
-    ))
+    from unittest import TestSuite, makeSuite
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestTestbrowser))
+    return suite

versions-prod.cfg

@@ -12,7 +12,6 @@ DocumentTemplate = 2.13.2
 ExtensionClass = 4.1.2
 five.globalrequest = 1.0
 funcsigs = 1.0.2
-mechanize = 0.2.5
 mock = 2.0.0
 Missing = 3.1
 MultiMapping = 3.0
@@ -82,7 +81,7 @@ zope.size = 4.1.0
 zope.structuredtext = 4.1.0
 zope.tal = 4.2.0
 zope.tales = 4.1.1
-zope.testbrowser = 4.0.4
+zope.testbrowser = 5.0.0
 zope.testing = 4.6.0
 zope.testrunner = 4.5.1
 zope.traversing = 4.1.0