Commit 00007c41 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Romain Courteaud

Store key only headers whose value is not stable.

parent 506522e1
...@@ -196,30 +196,27 @@ def checkHttpStatus( ...@@ -196,30 +196,27 @@ def checkHttpStatus(
) )
# Blacklisted, because of non stability # Blacklisted, because of non stability
# 'Date', 'Age', 'Expires' # 'Date'
header_list = [ header_list = [
# Redirect # Redirect
"Location", "Location",
# HTTP Range # HTTP Range
"Accept-Ranges", "Accept-Ranges",
# HTTP Cache # HTTP Cache
"Etag",
"Last-Modified",
"Vary", "Vary",
"Cache-Control", "Cache-Control",
"Set-Cookie", "WWW-Authenticate",
"WWW-Authenticate"
# gzip # gzip
"Content-Type", "Content-Type",
"Content-Encoding", "Content-Encoding",
"Content-Disposition" "Content-Disposition",
# Security # Security
"Content-Security-Policy", "Content-Security-Policy",
"Referrer-Policy", "Referrer-Policy",
"Strict-Transport-Policy", "Strict-Transport-Security",
"Feature-Policy", "Feature-Policy",
"X-Frame-Options", "X-Frame-Options",
"X-Content-Type-Options" "X-Content-Type-Options",
# CORS # CORS
"Access-Control-Allow-Origin", "Access-Control-Allow-Origin",
"Access-Control-Allow-Methods", "Access-Control-Allow-Methods",
...@@ -233,6 +230,20 @@ def checkHttpStatus( ...@@ -233,6 +230,20 @@ def checkHttpStatus(
if header_value is not None: if header_value is not None:
header_dict[header_key] = header_value header_dict[header_key] = header_value
# Store key only, because of non stability
# 'Etag', 'Last-Modified', 'Set-Cookie', 'Date', 'Age', 'Expires'
key_only_header_list = [
"Etag",
"Last-Modified",
"Set-Cookie",
"Age",
"Expires",
]
for header_key in key_only_header_list:
header_value = response.headers.get(header_key, None)
if header_value is not None:
header_dict[header_key] = True
logHttpStatus( logHttpStatus(
db, db,
ip, ip,
......
...@@ -862,7 +862,7 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -862,7 +862,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
httpretty.GET, httpretty.GET,
"http://127.0.0.1/foo?bar=1", "http://127.0.0.1/foo?bar=1",
status=418, status=418,
adding_headers={"Etag": "bar"}, adding_headers={"Etag": "bar", "Cache-Control": "public"},
) )
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
...@@ -884,7 +884,11 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -884,7 +884,11 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418 assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"} assert self.db.HttpCodeChange.get().http_header_dict == {
"Cache-Control": "public",
"Content-Type": "text/plain; charset=utf-8",
"Etag": True,
}
assert self.db.HttpCodeChange.get().status_id == status_id assert self.db.HttpCodeChange.get().status_id == status_id
def test_checkHttpStatus_https(self): def test_checkHttpStatus_https(self):
...@@ -894,7 +898,10 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -894,7 +898,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
with mock.patch("surykatka.http.request") as mock_request: with mock.patch("surykatka.http.request") as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"} mock_request.return_value.headers = {
"Etag": "foobar",
"Cache-Control": "public",
}
checkHttpStatus(self.db, status_id, url, ip, bot_version) checkHttpStatus(self.db, status_id, url, ip, bot_version)
...@@ -918,7 +925,8 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -918,7 +925,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
# XXX No idea how to mock SSL # XXX No idea how to mock SSL
assert self.db.HttpCodeChange.get().status_code == 1 assert self.db.HttpCodeChange.get().status_code == 1
assert self.db.HttpCodeChange.get().http_header_dict == { assert self.db.HttpCodeChange.get().http_header_dict == {
"Etag": "foobar" "Cache-Control": "public",
"Etag": True,
} }
assert self.db.HttpCodeChange.get().status_id == status_id assert self.db.HttpCodeChange.get().status_id == status_id
...@@ -935,9 +943,12 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -935,9 +943,12 @@ class SurykatkaHttpTestCase(unittest.TestCase):
else: else:
raise NotImplementedError("Expected NotImplementedError") raise NotImplementedError("Expected NotImplementedError")
def __generateHeaderDict(self, header_list): def __generateHeaderDict(self, header_list, key_only=False):
result_dict = {} result_dict = {}
for header in header_list: for header in header_list:
if key_only:
result_dict[header] = True
else:
result_dict[header] = header + " bar" result_dict[header] = header + " bar"
return result_dict return result_dict
...@@ -952,23 +963,20 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -952,23 +963,20 @@ class SurykatkaHttpTestCase(unittest.TestCase):
# HTTP Range # HTTP Range
"Accept-Ranges", "Accept-Ranges",
# HTTP Cache # HTTP Cache
"Etag",
"Last-Modified",
"Vary", "Vary",
"Cache-Control", "Cache-Control",
"Set-Cookie", "WWW-Authenticate",
"WWW-Authenticate"
# gzip # gzip
"Content-Type", "Content-Type",
"Content-Encoding", "Content-Encoding",
"Content-Disposition" "Content-Disposition",
# Security # Security
"Content-Security-Policy", "Content-Security-Policy",
"Referrer-Policy", "Referrer-Policy",
"Strict-Transport-Policy", "Strict-Transport-Security",
"Feature-Policy", "Feature-Policy",
"X-Frame-Options", "X-Frame-Options",
"X-Content-Type-Options" "X-Content-Type-Options",
# CORS # CORS
"Access-Control-Allow-Origin", "Access-Control-Allow-Origin",
"Access-Control-Allow-Methods", "Access-Control-Allow-Methods",
...@@ -976,13 +984,19 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -976,13 +984,19 @@ class SurykatkaHttpTestCase(unittest.TestCase):
"Access-Control-Allow-Headers", "Access-Control-Allow-Headers",
"Access-Control-Expose-Headers", "Access-Control-Expose-Headers",
] ]
blacklist_header_list = [ key_only_header_list = [
"Etag",
"Last-Modified",
"Set-Cookie",
"Age", "Age",
"Date",
"Expires", "Expires",
]
blacklist_header_list = [
"Foo", "Foo",
"Date",
] ]
header_dict = self.__generateHeaderDict(whitelist_header_list) header_dict = self.__generateHeaderDict(whitelist_header_list)
header_dict.update(self.__generateHeaderDict(key_only_header_list))
header_dict.update(self.__generateHeaderDict(blacklist_header_list)) header_dict.update(self.__generateHeaderDict(blacklist_header_list))
httpretty.register_uri( httpretty.register_uri(
...@@ -1011,9 +1025,16 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -1011,9 +1025,16 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418 assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == self.__generateHeaderDict( expected_http_header_dict = self.__generateHeaderDict(
whitelist_header_list whitelist_header_list
) )
expected_http_header_dict.update(
self.__generateHeaderDict(key_only_header_list, key_only=True)
)
assert (
self.db.HttpCodeChange.get().http_header_dict
== expected_http_header_dict
)
assert self.db.HttpCodeChange.get().status_id == status_id assert self.db.HttpCodeChange.get().status_id == status_id
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment