Commit 340a7609 authored by Jérome Perrin's avatar Jérome Perrin

software/*: version up urllib3, requests and dependencies

Several softwares were using fixed versions, but versions are also defined in
stack/slapos.cfg, so let's use from stack/slapos.cfg which will simplify
version management.

This urllib3 has visible changes for us, it no longer send the request path
"as is", but normalizes the . and .. which is not what we want in
caddy-frontend test, so we adjust the test to use prepared requests, which is
the recommended approach for this kind of low level needs. Since prepared
requests does not set default user agent "python-requests", tests checking that
user agent is in log files have been adjusted to use a dedicated test user agent
header.

Another change is that certificates are now verified by default, so we adjust
seleniumserver tests to disable certificate validation. jstestnode, already
had a patch regarding certificate validation, because we wanted certificates
to be validated before it was the default. We just update this patch a bit
because we want to keep the optional behaviours of validating against a
specific CA or to disable validation.
parent 0f5d6bd7
Pipeline #13508 failed with stage
in 0 seconds
...@@ -228,15 +228,6 @@ mode = 0644 ...@@ -228,15 +228,6 @@ mode = 0644
[versions] [versions]
# Modern KeDiFa requires zc.lockfile # Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4 zc.lockfile = 1.4
# Versions pinned for kedifa need urllib3 >= 1.18
urllib3 = 1.24
requests = 2.20.0
certifi = 2018.10.15
idna = 2.7
chardet = 3.0.4
# ipaddress is patching IPAddress so IPv6 match works
ipaddress = 1.0.22
# Versions pinned for kedifa need urllib3 >= 1.18
validators = 0.12.2 validators = 0.12.2
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
......
...@@ -419,13 +419,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT, ...@@ -419,13 +419,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
try: try:
add_custom_dns(domain, port, TEST_IP) add_custom_dns(domain, port, TEST_IP)
socket.getaddrinfo = new_getaddrinfo socket.getaddrinfo = new_getaddrinfo
return session.get( # Use a prepared request, to disable path normalization.
'https://%s:%s/%s' % (domain, port, path), # We need this because some test checks requests with paths like
verify=False, # /test-path/deep/.././deeper but we don't want the client to send
allow_redirects=False, # /test-path/deeper
# See also https://github.com/psf/requests/issues/5289
url = 'https://%s:%s/%s' % (domain, port, path)
req = requests.Request(
method='GET',
url=url,
headers=headers, headers=headers,
cookies=cookies cookies=cookies,
) )
prepped = req.prepare()
prepped.url = url
return session.send(prepped, verify=False, allow_redirects=False)
finally: finally:
socket.getaddrinfo = socket_getaddrinfo socket.getaddrinfo = socket_getaddrinfo
...@@ -447,11 +455,13 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT, ...@@ -447,11 +455,13 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
new_source = source.SourceAddressAdapter(source_ip) new_source = source.SourceAddressAdapter(source_ip)
session.mount('http://', new_source) session.mount('http://', new_source)
session.mount('https://', new_source) session.mount('https://', new_source)
return session.get(
'http://%s:%s/%s' % (TEST_IP, port, path), # Use a prepared request, to disable path normalization.
headers=headers, url = 'http://%s:%s/%s' % (TEST_IP, port, path)
allow_redirects=False, req = requests.Request(method='GET', url=url, headers=headers)
) prepped = req.prepare()
prepped.url = url
return session.send(prepped, allow_redirects=False)
class TestHandler(BaseHTTPRequestHandler): class TestHandler(BaseHTTPRequestHandler):
...@@ -1821,6 +1831,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1821,6 +1831,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
headers={ headers={
'Timeout': '10', # more than default backend-connect-timeout == 5 'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip', 'Accept-Encoding': 'gzip',
'User-Agent': 'TEST USER AGENT',
} }
) )
...@@ -1854,7 +1865,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1854,7 +1865,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \ log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \ r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \ r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
r'\d+ "-" "python-requests.*" \d+' r'\d+ "-" "TEST USER AGENT" \d+'
self.assertRegexpMatches( self.assertRegexpMatches(
open(log_file, 'r').readlines()[-1], open(log_file, 'r').readlines()[-1],
......
...@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase): ...@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
).json() ).json()
self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4') self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth'] balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
with self.assertRaises(OpenSSL.SSL.Error): with self.assertRaisesRegexp(Exception, "certificate required"):
requests.get( requests.get(
balancer_url, balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'}, headers={'X-Forwarded-For': '1.2.3.4'},
......
...@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05 ...@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
[template-runTestSuite] [template-runTestSuite]
filename = runTestSuite.in filename = runTestSuite.in
md5sum = bb3f053b6cdb0a8888e9d32e63085ed5 md5sum = 73ef758825563b7a6a1d660d4d5229b8
...@@ -97,8 +97,8 @@ def main(): ...@@ -97,8 +97,8 @@ def main():
executable_path='${geckodriver:location}') executable_path='${geckodriver:location}')
else: else:
assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target']) assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
# use a remote connection which verifies TLS certificate # use a remote connection which optionally verifies TLS certificate
# workaround for https://github.com/SeleniumHQ/selenium/issues/6534 # and optionally against a user provided CA.
executor = RemoteConnection(test_runner['server-url'], keep_alive=True) executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
cert_reqs = 'CERT_REQUIRED' cert_reqs = 'CERT_REQUIRED'
ca_certs = certifi.where() ca_certs = certifi.where()
......
...@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in ...@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in
output = ${buildout:directory}/runTestSuite.in output = ${buildout:directory}/runTestSuite.in
[versions] [versions]
selenium = 3.14.1 selenium = 3.14.1
urllib3 = 1.24
certifi = 2018.10.15
...@@ -81,12 +81,7 @@ surykatka = 0.5.0 ...@@ -81,12 +81,7 @@ surykatka = 0.5.0
# For surykatka 0.5.0 # For surykatka 0.5.0
click = 7.0 click = 7.0
certifi = 2019.11.28
chardet = 3.0.4
dnspython = 1.16.0 dnspython = 1.16.0
forcediphttpsadapter = 1.0.1 forcediphttpsadapter = 1.0.1
idna = 2.8
miniupnpc = 2.0.2 miniupnpc = 2.0.2
peewee = 3.13.1 peewee = 3.13.1
requests = 2.22.0
urllib3 = 1.25.8
...@@ -46,8 +46,10 @@ from PIL import Image ...@@ -46,8 +46,10 @@ from PIL import Image
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.remote.remote_connection import RemoteConnection
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
import urllib3
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
...@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase): ...@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
self.assertEqual('selenium', parsed.username) self.assertEqual('selenium', parsed.username)
self.assertTrue(parsed.password) self.assertTrue(parsed.password)
# XXX we are using a self signed certificate, but selenium 3.141.0 does
# not expose API to ignore certificate verification
executor = RemoteConnection(webdriver_url, keep_alive=True)
executor._conn = urllib3.PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
driver = webdriver.Remote( driver = webdriver.Remote(
command_executor=webdriver_url, command_executor=executor,
desired_capabilities=DesiredCapabilities.CHROME) desired_capabilities=DesiredCapabilities.CHROME)
driver.get(self.server_url) driver.get(self.server_url)
......
...@@ -298,16 +298,7 @@ selenium = 3.141.0 ...@@ -298,16 +298,7 @@ selenium = 3.141.0
# Patched eggs # Patched eggs
PyPDF2 = 1.26.0+SlapOSPatched001 PyPDF2 = 1.26.0+SlapOSPatched001
# modern versions for SSL fixed support # caucase and its dependencies
requests = 2.20.0
certifi = 2018.10.15
idna = 2.7
chardet = 3.0.4
# ipaddress is patching IPAddress so IPv6 match works
ipaddress = 1.0.22
# cacuase and its dependencies
caucase = 0.9.4 caucase = 0.9.4
pem = 18.2.0 pem = 18.2.0
PyJWT = 1.6.4 PyJWT = 1.6.4
...@@ -315,10 +306,6 @@ PyJWT = 1.6.4 ...@@ -315,10 +306,6 @@ PyJWT = 1.6.4
# Django 1.11 is python 2 compatible # Django 1.11 is python 2 compatible
Django = 1.11 Django = 1.11
# Required by:
# selenium==3.141.0
urllib3 = 1.24.1
mock = 2.0.0 mock = 2.0.0
testfixtures = 6.11 testfixtures = 6.11
funcsigs = 1.0.2 funcsigs = 1.0.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment