software/*: version up urllib3, requests and dependencies

Several softwares were using fixed versions, but versions are also defined in
stack/slapos.cfg, so let's use from stack/slapos.cfg which will simplify
version management.

This urllib3 has visible changes for us, it no longer send the request path
"as is", but normalizes the . and .. which is not what we want in
caddy-frontend test, so we adjust the test to use prepared requests, which is
the recommended approach for this kind of low level needs. Since prepared
requests does not set default user agent "python-requests", tests checking that
user agent is in log files have been adjusted to use a dedicated test user agent
header.

Another change is that certificates are now verified by default, so we adjust
seleniumserver tests to disable certificate validation. jstestnode, already
had a patch regarding certificate validation, because we wanted certificates
to be validated before it was the default. We just update this patch a bit
because we want to keep the optional behaviours of validating against a
specific CA or to disable validation.

software/caddy-frontend/software.cfg

@@ -228,15 +228,6 @@ mode = 0644
 [versions]
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4
-# Versions pinned for kedifa need urllib3 >= 1.18
-urllib3 = 1.24
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-# Versions pinned for kedifa need urllib3 >= 1.18
 
 validators = 0.12.2
 PyRSS2Gen = 1.1

software/caddy-frontend/test/test.py

@@ -419,13 +419,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
   try:
     add_custom_dns(domain, port, TEST_IP)
     socket.getaddrinfo = new_getaddrinfo
-    return session.get(
-      'https://%s:%s/%s' % (domain, port, path),
-      verify=False,
-      allow_redirects=False,
+    # Use a prepared request, to disable path normalization.
+    # We need this because some test checks requests with paths like
+    # /test-path/deep/.././deeper but we don't want the client to send
+    # /test-path/deeper
+    # See also https://github.com/psf/requests/issues/5289
+    url = 'https://%s:%s/%s' % (domain, port, path)
+    req = requests.Request(
+        method='GET',
+        url=url,
         headers=headers,
-      cookies=cookies
+        cookies=cookies,
     )
+    prepped = req.prepare()
+    prepped.url = url
+    return session.send(prepped, verify=False, allow_redirects=False)
   finally:
     socket.getaddrinfo = socket_getaddrinfo
 
@@ -447,11 +455,13 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
     new_source = source.SourceAddressAdapter(source_ip)
     session.mount('http://', new_source)
     session.mount('https://', new_source)
-  return session.get(
-    'http://%s:%s/%s' % (TEST_IP, port, path),
-    headers=headers,
-    allow_redirects=False,
-  )
+
+  # Use a prepared request, to disable path normalization.
+  url = 'http://%s:%s/%s' % (TEST_IP, port, path)
+  req = requests.Request(method='GET', url=url, headers=headers)
+  prepped = req.prepare()
+  prepped.url = url
+  return session.send(prepped, allow_redirects=False)
 
 
 class TestHandler(BaseHTTPRequestHandler):
@@ -1821,6 +1831,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       headers={
         'Timeout': '10',  # more than default backend-connect-timeout == 5
         'Accept-Encoding': 'gzip',
+        'User-Agent': 'TEST USER AGENT',
       }
     )
 
@@ -1854,7 +1865,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
                  r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
                  r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
-                 r'\d+ "-" "python-requests.*" \d+'
+                 r'\d+ "-" "TEST USER AGENT" \d+'
 
     self.assertRegexpMatches(
       open(log_file, 'r').readlines()[-1],

software/erp5/test/test/test_balancer.py

@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
     ).json()
     self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
     balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
-    with self.assertRaises(OpenSSL.SSL.Error):
+    with self.assertRaisesRegexp(Exception, "certificate required"):
       requests.get(
         balancer_url,
         headers={'X-Forwarded-For': '1.2.3.4'},

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = bb3f053b6cdb0a8888e9d32e63085ed5
+md5sum = 73ef758825563b7a6a1d660d4d5229b8

software/jstestnode/runTestSuite.in

@@ -97,8 +97,8 @@ def main():
             executable_path='${geckodriver:location}')
       else:
         assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
-        # use a remote connection which verifies TLS certificate
-        # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
+        # use a remote connection which optionally verifies TLS certificate
+        # and optionally against a user provided CA.
         executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
         cert_reqs = 'CERT_REQUIRED'
         ca_certs = certifi.where()

software/jstestnode/software.cfg

@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in
 output = ${buildout:directory}/runTestSuite.in
 
 [versions]
-
 selenium = 3.14.1
-urllib3 = 1.24
-certifi = 2018.10.15

software/monitor/software.cfg

@@ -81,12 +81,7 @@ surykatka = 0.5.0
 
 # For surykatka 0.5.0
 click = 7.0
-certifi = 2019.11.28
-chardet = 3.0.4
 dnspython = 1.16.0
 forcediphttpsadapter = 1.0.1
-idna = 2.8
 miniupnpc = 2.0.2
 peewee = 3.13.1
-requests = 2.22.0
-urllib3 = 1.25.8

software/seleniumserver/test/test.py

@@ -46,8 +46,10 @@ from PIL import Image
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.remote.remote_connection import RemoteConnection
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
+import urllib3
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
     self.assertEqual('selenium', parsed.username)
     self.assertTrue(parsed.password)
 
+    # XXX we are using a self signed certificate, but selenium 3.141.0 does
+    # not expose API to ignore certificate verification
+    executor = RemoteConnection(webdriver_url, keep_alive=True)
+    executor._conn = urllib3.PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
+
     driver = webdriver.Remote(
-        command_executor=webdriver_url,
+        command_executor=executor,
         desired_capabilities=DesiredCapabilities.CHROME)
 
     driver.get(self.server_url)

software/slapos-sr-testing/software.cfg

@@ -298,16 +298,7 @@ selenium = 3.141.0
 # Patched eggs
 PyPDF2 = 1.26.0+SlapOSPatched001
 
-# modern versions for SSL fixed support
-requests = 2.20.0
-certifi = 2018.10.15
-idna = 2.7
-chardet = 3.0.4
-
-# ipaddress is patching IPAddress so IPv6 match works
-ipaddress = 1.0.22
-
-# cacuase and its dependencies
+# caucase and its dependencies
 caucase = 0.9.4
 pem = 18.2.0
 PyJWT = 1.6.4
@@ -315,10 +306,6 @@ PyJWT = 1.6.4
 # Django 1.11 is python 2 compatible
 Django = 1.11
 
-# Required by:
-# selenium==3.141.0
-urllib3 = 1.24.1
-
 mock = 2.0.0
 testfixtures = 6.11
 funcsigs = 1.0.2