Commit 00007c41 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Romain Courteaud

Store key only headers whose value is not stable.

parent 506522e1
......@@ -196,30 +196,27 @@ def checkHttpStatus(
)
# Blacklisted, because of non stability
# 'Date', 'Age', 'Expires'
# 'Date'
header_list = [
# Redirect
"Location",
# HTTP Range
"Accept-Ranges",
# HTTP Cache
"Etag",
"Last-Modified",
"Vary",
"Cache-Control",
"Set-Cookie",
"WWW-Authenticate"
"WWW-Authenticate",
# gzip
"Content-Type",
"Content-Encoding",
"Content-Disposition"
"Content-Disposition",
# Security
"Content-Security-Policy",
"Referrer-Policy",
"Strict-Transport-Policy",
"Strict-Transport-Security",
"Feature-Policy",
"X-Frame-Options",
"X-Content-Type-Options"
"X-Content-Type-Options",
# CORS
"Access-Control-Allow-Origin",
"Access-Control-Allow-Methods",
......@@ -233,6 +230,20 @@ def checkHttpStatus(
if header_value is not None:
header_dict[header_key] = header_value
# Store key only, because of non stability
# 'Etag', 'Last-Modified', 'Set-Cookie', 'Date', 'Age', 'Expires'
key_only_header_list = [
"Etag",
"Last-Modified",
"Set-Cookie",
"Age",
"Expires",
]
for header_key in key_only_header_list:
header_value = response.headers.get(header_key, None)
if header_value is not None:
header_dict[header_key] = True
logHttpStatus(
db,
ip,
......
......@@ -862,7 +862,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
httpretty.GET,
"http://127.0.0.1/foo?bar=1",
status=418,
adding_headers={"Etag": "bar"},
adding_headers={"Etag": "bar", "Cache-Control": "public"},
)
status_id = logStatus(self.db, "foo")
......@@ -884,7 +884,11 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"}
assert self.db.HttpCodeChange.get().http_header_dict == {
"Cache-Control": "public",
"Content-Type": "text/plain; charset=utf-8",
"Etag": True,
}
assert self.db.HttpCodeChange.get().status_id == status_id
def test_checkHttpStatus_https(self):
......@@ -894,7 +898,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_id = logStatus(self.db, "foo")
with mock.patch("surykatka.http.request") as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_request.return_value.headers = {
"Etag": "foobar",
"Cache-Control": "public",
}
checkHttpStatus(self.db, status_id, url, ip, bot_version)
......@@ -918,7 +925,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
# XXX No idea how to mock SSL
assert self.db.HttpCodeChange.get().status_code == 1
assert self.db.HttpCodeChange.get().http_header_dict == {
"Etag": "foobar"
"Cache-Control": "public",
"Etag": True,
}
assert self.db.HttpCodeChange.get().status_id == status_id
......@@ -935,10 +943,13 @@ class SurykatkaHttpTestCase(unittest.TestCase):
else:
raise NotImplementedError("Expected NotImplementedError")
def __generateHeaderDict(self, header_list):
def __generateHeaderDict(self, header_list, key_only=False):
result_dict = {}
for header in header_list:
result_dict[header] = header + " bar"
if key_only:
result_dict[header] = True
else:
result_dict[header] = header + " bar"
return result_dict
@httpretty.activate
......@@ -952,23 +963,20 @@ class SurykatkaHttpTestCase(unittest.TestCase):
# HTTP Range
"Accept-Ranges",
# HTTP Cache
"Etag",
"Last-Modified",
"Vary",
"Cache-Control",
"Set-Cookie",
"WWW-Authenticate"
"WWW-Authenticate",
# gzip
"Content-Type",
"Content-Encoding",
"Content-Disposition"
"Content-Disposition",
# Security
"Content-Security-Policy",
"Referrer-Policy",
"Strict-Transport-Policy",
"Strict-Transport-Security",
"Feature-Policy",
"X-Frame-Options",
"X-Content-Type-Options"
"X-Content-Type-Options",
# CORS
"Access-Control-Allow-Origin",
"Access-Control-Allow-Methods",
......@@ -976,13 +984,19 @@ class SurykatkaHttpTestCase(unittest.TestCase):
"Access-Control-Allow-Headers",
"Access-Control-Expose-Headers",
]
blacklist_header_list = [
key_only_header_list = [
"Etag",
"Last-Modified",
"Set-Cookie",
"Age",
"Date",
"Expires",
]
blacklist_header_list = [
"Foo",
"Date",
]
header_dict = self.__generateHeaderDict(whitelist_header_list)
header_dict.update(self.__generateHeaderDict(key_only_header_list))
header_dict.update(self.__generateHeaderDict(blacklist_header_list))
httpretty.register_uri(
......@@ -1011,9 +1025,16 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == self.__generateHeaderDict(
expected_http_header_dict = self.__generateHeaderDict(
whitelist_header_list
)
expected_http_header_dict.update(
self.__generateHeaderDict(key_only_header_list, key_only=True)
)
assert (
self.db.HttpCodeChange.get().http_header_dict
== expected_http_header_dict
)
assert self.db.HttpCodeChange.get().status_id == status_id
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment