Skip to content

slapos
slapos
Commit 8094a396 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

XXX rapid-cdn: Rewrite test backend 

XXX:
 * improve it more
 * adapt tests

Test backend needs to mimic all possible real backend cases.
parent 390b2e97
Show whitespace changes
Inline Side-by-side
Showing with 182 additions and 188 deletions
software/rapid-cdn/test/test.py
View file @ 8094a396
......@@ -40,8 +40,6 @@ from socketserver import ThreadingMixIn
import time
import tempfile
import ipaddress
import io
import gzip
import base64
import re
from slapos.recipe.librecipe import generateHashFromFiles
......@@ -51,8 +49,6 @@ import socket
import sys
import logging
import lzma
import random
import string
from slapos.slap.standalone import SlapOSNodeInstanceError
import caucase.client
import caucase.utils
......@@ -242,6 +238,13 @@ def subprocess_output(*args, **kwargs):
return subprocess_status_output(*args, **kwargs)[1]
def setUpHeaders(header_list_list):
result = http.client.HTTPMessage()
for header, value in header_list_list:
result.add_header(header, value)
return result
mimikra = Recurls()
......@@ -499,20 +502,29 @@ class TestDataMixin(object):
self.assertTestData(json_data, data_replacement_dict=data_replacement_dict)
def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
headers=None, source_ip=SOURCE_IP):
def fakeSetupHeaders(headers):
if headers is None:
headers = {}
headers = http.client.HTTPMessage()
default_header_dict = {
# workaround request problem of setting Accept-Encoding
# https://github.com/requests/requests/issues/2234
headers.setdefault('Accept-Encoding', 'dummy')
'Accept-Encoding': 'dummy',
# Headers to tricks the whole system, like rouge user would do
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
'X-Forwarded-For': '192.168.0.1',
'X-Forwarded-Proto': 'irc',
'X-Forwarded-Port': '17',
# Expose some Via to show how nicely it arrives to the backend
headers.setdefault('Via', 'http/1.1 clientvia')
'Via': 'http/1.1 clientvia'
}
for header_name, header_value in default_header_dict.items():
if header_name not in headers:
headers.add_header(header_name, header_value)
return headers
def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
headers=None, source_ip=SOURCE_IP):
headers = fakeSetupHeaders(headers)
url = 'https://%s:%s/%s' % (domain, port, path)
return mimikra.get(
......@@ -535,18 +547,9 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
def fakeHTTPResult(domain, path, port=HTTP_PORT,
headers=None, source_ip=SOURCE_IP):
if headers is None:
headers = {}
# workaround request problem of setting Accept-Encoding
# https://github.com/requests/requests/issues/2234
headers.setdefault('Accept-Encoding', 'dummy')
# Headers to tricks the whole system, like rouge user would do
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
# Expose some Via to show how nicely it arrives to the backend
headers.setdefault('Via', 'http/1.1 clientvia')
headers['Host'] = '%s:%s' % (domain, port)
headers = fakeSetupHeaders(headers)
if 'Host' not in headers:
headers.add_header('Host', '%s:%s' % (domain, port))
url = 'http://%s:%s/%s' % (TEST_IP, port, path)
return mimikra.get(
url,
......@@ -560,12 +563,25 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
)
class ConfigurationReplyEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, bytes):
return '<%i>' % (len(obj))
elif isinstance(obj, http.client.HTTPMessage):
reply = ''
for k, v in obj.items():
reply += '%r: %r' % (k, v)
return reply
return json.JSONEncoder.default(self, obj)
class TestHandler(BaseHTTPRequestHandler):
identification = None
configuration = {}
# override Server header response
server_version = "TestBackend"
sys_version = ""
DEFAULT_CONFIGURATION = {
'Status-Code': '200',
'Protocol-Version': 'HTTP/1.0',
'Timeout': '0',
}
log_message = logging.getLogger(__name__ + '.TestHandler').info
......@@ -580,127 +596,63 @@ class TestHandler(BaseHTTPRequestHandler):
self.end_headers()
self.wfile.write(json.dumps({self.path: config}, indent=2))
def do_PUT(self):
incoming_config = {}
for key, value in list(self.headers.items()):
if key.startswith('X-'):
incoming_config[key] = value
config = {
'status_code': incoming_config.pop('X-Reply-Status-Code', '200')
}
prefix = 'X-Reply-Header-'
length = len(prefix)
for key in list(incoming_config.keys()):
if key.startswith(prefix):
header = '-'.join([q.capitalize() for q in key[length:].split('-')])
config[header] = incoming_config.pop(key)
if 'X-Reply-Body' in incoming_config:
config['Body'] = base64.b64decode(
incoming_config.pop('X-Reply-Body')).decode()
config['X-Drop-Header'] = incoming_config.pop('X-Drop-Header', None)
self.configuration[self.path] = config
def do_CONFIG(self):
config = self.DEFAULT_CONFIGURATION.copy()
incoming_headers = http.client.HTTPMessage()
config_header = 'X-Config-'
config_header_header = 'X-Config-Reply-Header-'
for header_name, header_value in self.headers.items():
if header_name.startswith(config_header_header):
incoming_headers.add_header(
header_name[len(config_header_header):], header_value)
elif header_name.startswith(config_header):
config[header_name[len(config_header):]] = header_value
config['Body'] = self.rfile.read(int(self.headers.get(
'Content-Length', '0')))
self.send_response(201)
self.send_header("Content-Type", "application/json")
self.end_headers()
reply = {self.path: config}
if incoming_config:
reply['unknown_config'] = incoming_config
self.wfile.write(json.dumps(reply, indent=2).encode())
self.configuration[self.path] = {
'headers': incoming_headers,
'configuration': config
}
reply = {self.path: dict(self.configuration[self.path])}
self.wfile.write(json.dumps(
reply, indent=2, cls=ConfigurationReplyEncoder).encode())
def do_POST(self):
return self.do_GET()
def do_GET(self):
config = self.configuration.get(self.path, None)
if config is not None:
config = config.copy()
response = config.pop('Body', None)
status_code = int(config.pop('status_code'))
timeout = int(config.pop('Timeout', '0'))
compress = int(config.pop('Compress', '0'))
drop_header_list = []
for header in (config.pop('X-Drop-Header') or '').split():
drop_header_list.append(header)
header_dict = config
else:
drop_header_list = []
for header in (self.headers.get('x-drop-header') or '').split():
drop_header_list.append(header)
response = None
status_code = 200
timeout = int(self.headers.get('timeout', '0'))
if 'x-maximum-timeout' in self.headers:
maximum_timeout = int(self.headers['x-maximum-timeout'])
timeout = random.randrange(maximum_timeout)
if 'x-response-size' in self.headers:
min_response, max_response = [
int(q) for q in self.headers['x-response-size'].split(' ')]
reponse_size = random.randrange(min_response, max_response)
response = ''.join(
random.choice(string.lowercase) for x in range(reponse_size))
compress = int(self.headers.get('compress', '0'))
header_dict = {}
prefix = 'x-reply-header-'
length = len(prefix)
for key, value in list(self.headers.items()):
if key.startswith(prefix):
header = '-'.join([q.capitalize() for q in key[length:].split('-')])
header_dict[header] = value.strip()
if response is None:
if 'x-reply-body' not in self.headers:
headers_dict = dict()
for header in list(self.headers.keys()):
content = self.headers.get_all(header)
if len(content) == 0:
headers_dict[header] = None
elif len(content) == 1:
headers_dict[header] = content[0]
else:
headers_dict[header] = content
response = {
'Path': self.path,
'Incoming Headers': headers_dict
}
response = json.dumps(response, indent=2)
else:
response = base64.b64decode(self.headers['x-reply-body'])
time.sleep(timeout)
self.send_response_only(status_code)
self.send_header('Server', self.server_version)
for key, value in list(header_dict.items()):
self.send_header(key, value)
if self.identification is not None:
self.send_header('X-Backend-Identification', self.identification)
if 'Content-Type' not in drop_header_list:
self.send_header("Content-Type", "application/json")
if 'Set-Cookie' not in drop_header_list:
self.send_header('Set-Cookie', 'secured=value;secure')
self.send_header('Set-Cookie', 'nonsecured=value')
if 'Via' not in drop_header_list:
self.send_header('Via', 'http/1.1 backendvia')
if compress:
self.send_header('Content-Encoding', 'gzip')
out = io.BytesIO()
# compress with level 0, to find out if in the middle someting would
# like to alter the compression
with gzip.GzipFile(fileobj=out, mode="wb", compresslevel=0) as f:
f.write(response.encode())
response = out.getvalue()
self.send_header('Backend-Content-Length', len(response))
if 'Content-Length' not in drop_header_list:
if config is None:
self.send_response(404)
response = json.dumps({'path': self.path}, indent=2).encode()
self.send_header('Content-Length', len(response))
self.send_header('Content-Type', 'application/json')
self.end_headers()
if getattr(response, 'encode', None) is not None:
response = response.encode()
self.wfile.write(response)
return
self.protocol_version = config['configuration']['Protocol-Version']
time.sleep(int(config['configuration']['Timeout']))
self.send_response_only(int(config['configuration']['Status-Code']))
for header, value in config['headers'].items():
for header_type in ['Date', 'Last-Modified']:
if header == header_type:
if value == 'now':
value = self.date_time_string()
if header == 'Expires':
if value.startswith('delta:'):
value = self.date_time_string(
time.time() + float(value.split(':')[1])
)
if header == 'Content-Length':
if value == 'calculate':
value = '%s' % (len(config['configuration']['Body']),)
self.send_header(header, value)
self.end_headers()
self.wfile.write(config['configuration']['Body'])
class HttpFrontendTestCase(SlapOSInstanceTestCase):
......@@ -984,49 +936,66 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
client_version = self.max_client_version
if alt_svc is None:
alt_svc = self.alt_svc
headers = result.headers.copy()
self.assertKeyWithPop('Content-Length', headers)
if 'Connection' in headers and headers[
'Connection'].lower() == 'keep-alive':
headers.pop('Connection')
pop_header_list = []
def assertSingleHeader(header):
value_list = result.headers.get_all(header, [])
self.assertEqual(1, len(value_list))
return value_list[0]
def assertAndPopSingleHeader(header):
pop_header_list.append(header.lower())
return assertSingleHeader(header)
assertAndPopSingleHeader('Content-Length')
if 'Connection' in result.headers:
if assertSingleHeader().lower() == 'keep-alive':
pop_header_list.append('Connection'.lower())
if alt_svc:
self.assertEqual(
'h3=":%s"; ma=3600' % (HTTPS_PORT,),
headers.pop('Alt-Svc', '')
assertAndPopSingleHeader('Alt-Svc')
)
self.assertEqual(
'%s:quic' % (HTTPS_PORT,),
headers.pop('Alternate-Protocol', '')
assertAndPopSingleHeader('Alternate-Protocol')
)
if backend_reached:
self.assertEqual('TestBackend', headers.pop('Server', ''))
self.assertKeyWithPop('Date', headers)
self.assertEqual('TestBackend', assertAndPopSingleHeader('Server'))
assertAndPopSingleHeader('Date')
via_id = '%s-%s' % (
self.node_information_dict['node-id'],
list(self.node_information_dict['version-hash-history'].keys())[0])
if via:
self.assertIn('Via', headers)
pop_header_list.append('Via'.lower())
via = ' '.join(result.headers.get_all('Via'))
if cached:
self.assertEqual(
'http/1.1 backendvia, '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s, '
'http/1.0 rapid-cdn-cache-%(via_id)s, '
'http/1.1 backendvia '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s, ' # ATS adds to existing
# header, so ","
'http/1.0 rapid-cdn-cache-%(via_id)s '
'HTTP/%(client_version)s rapid-cdn-frontend-%(via_id)s' % dict(
via_id=via_id, client_version=client_version),
headers.pop('Via')
via
)
else:
self.assertEqual(
'http/1.1 backendvia, '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s, '
'http/1.1 backendvia '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s '
'HTTP/%(client_version)s rapid-cdn-frontend-%(via_id)s' % dict(
via_id=via_id, client_version=client_version),
headers.pop('Via')
via
)
else:
self.assertNotIn('Via', headers)
self.assertNotIn('Via', result.headers)
headers = http.client.HTTPMessage()
for header, value in result.headers.items():
if header not in pop_header_list:
headers.add_header(header, value)
return headers
def assertLogAccessUrlWithPop(self, parameter_dict):
......@@ -2364,14 +2333,22 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
self.backend_url, self.backend_url)],
}
)
path = '/test-path/deep/.././deeper' * 250
backend_url = self.getSlaveParameterDictDict()['Url']['url']
config_result = mimikra.config(
backend_url + path,
headers=setUpHeaders([
('X-Config-Timeout', '10')
])
)
self.assertEqual(config_result.status_code, http.client.CREATED)
result = fakeHTTPSResult(
parameter_dict['domain'],
'/test-path/deep/.././deeper' * 250,
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
'User-Agent': 'TEST USER AGENT',
}
path,
headers=setUpHeaders([
('Accept-Encoding', 'gzip'),
('User-Agent', 'TEST USER AGENT'),
])
)
self.assertEqual(
......@@ -2512,6 +2489,15 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
def test_auth_to_backend(self):
parameter_dict = self.assertSlaveBase('auth-to-backend')
path = 'test-path/deep/.././deeper'
backend_url = self.getSlaveParameterDictDict()['auth-to-backend']['url']
config_result = mimikra.config(
backend_url + path,
headers=setUpHeaders([
('X-Config-Timeout', '10')
])
)
self.assertEqual(config_result.status_code, http.client.CREATED)
self.startAuthenticatedServerProcess()
try:
# assert that you can't fetch nothing without key
......@@ -2524,11 +2510,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
# (so it means that auth to backend worked)
result = fakeHTTPSResult(
parameter_dict['domain'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
path,
headers=setUpHeaders([
('Accept-Encoding', 'gzip'),
])
)
self.assertEqual(
......@@ -2561,6 +2546,16 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
def test_auth_to_backend_not_configured(self):
parameter_dict = self.assertSlaveBase('auth-to-backend-not-configured')
path = 'test-path/deep/.././deeper'
backend_url = self.getSlaveParameterDictDict()[
'auth-to-backend-not-configured']['url']
config_result = mimikra.config(
backend_url + path,
headers=setUpHeaders([
('X-Config-Timeout', '10')
])
)
self.assertEqual(config_result.status_code, http.client.CREATED)
self.startAuthenticatedServerProcess()
try:
# assert that you can't fetch nothing without key
......@@ -2573,11 +2568,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
# (so it means that auth to backend worked)
result = fakeHTTPSResult(
parameter_dict['domain'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
path,
headers=setUpHeaders([
('Accept-Encoding', 'gzip'),
])
)
self.assertEqual(
......@@ -4061,7 +4055,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
def configureResult(status_code, body):
backend_url = self.getSlaveParameterDictDict()['enable_cache']['url']
result = mimikra.put(backend_url + path, headers={
result = mimikra.config(backend_url + path, headers={
'X-Reply-Header-Cache-Control': 'max-age=%s, public' % (max_age,),
'X-Reply-Status-Code': status_code,
'X-Reply-Body': base64.b64encode(body.encode()).decode(),
......@@ -4832,7 +4826,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
normal_path = 'normal'
with_date_path = 'with_date'
specific_date = 'Fri, 07 Dec 2001 00:00:00 GMT'
result_configure = mimikra.put(
result_configure = mimikra.config(
backend_url + '/' + with_date_path, headers={
'X-Reply-Header-Date': specific_date
})
......@@ -7162,7 +7156,7 @@ backend _health-check-default-http
'failover-url?a=b&c=',
'failover-https-url?a=b&c='
]:
result = mimikra.put(
result = mimikra.config(
self.backend_url + url + path,
headers={
'X-Reply-Status-Code': '503',
......@@ -7173,7 +7167,7 @@ backend _health-check-default-http
def configureResult(status_code, body):
backend_url = self.getSlaveParameterDictDict()[
'health-check-failover-url']['https-url']
result = mimikra.put(
result = mimikra.config(
'/'.join([backend_url, cached_path]),
headers={
'X-Reply-Header-Cache-Control': 'max-age=%s, public' % (max_age,),
......@@ -7204,14 +7198,14 @@ backend _health-check-default-http
checkResult(http.client.OK, body_200)
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, http.client.CREATED)
def restoreBackend():
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={})
......@@ -7274,7 +7268,7 @@ backend _health-check-default-http
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertNotIn('X-Backend-Identification', result.headers)
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
......@@ -7282,7 +7276,7 @@ backend _health-check-default-http
self.assertEqual(result.status_code, http.client.CREATED)
def restoreBackend():
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={})
......@@ -7321,7 +7315,7 @@ backend _health-check-default-http
self.assertNotIn('X-Backend-Identification', result.headers)
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
......@@ -7354,7 +7348,7 @@ backend _health-check-default-http
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
......@@ -7383,7 +7377,7 @@ backend _health-check-default-http
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
......@@ -7413,7 +7407,7 @@ backend _health-check-default-http
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = mimikra.put(
result = mimikra.config(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7