Skip to content

S
surykatka
Commit 916e497e authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Reformatted with black

parent 24a1e7d4
Hide whitespace changes
Inline Side-by-side
Showing with 223 additions and 28 deletions
src/surykatka/bot.py
View file @ 916e497e
...@@ -144,7 +144,14 @@ class WebBot: ...@@ -144,7 +144,14 @@ class WebBot:
for url in url_dict: for url in url_dict:
for ip in url_dict[url]: for ip in url_dict[url]:
checkHttpStatus( checkHttpStatus(
self._db, status_id, url, ip, __version__, timeout, elapsed_fast, elapsed_moderate self._db,
status_id,
url,
ip,
__version__,
timeout,
elapsed_fast,
elapsed_moderate,
) )
# XXX Check location header and check new url recursively # XXX Check location header and check new url recursively
# XXX Parse HTML, fetch found link, css, js, image # XXX Parse HTML, fetch found link, css, js, image
......
src/surykatka/http.py
View file @ 916e497e
...@@ -133,7 +133,9 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id): ...@@ -133,7 +133,9 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
(previous_entry is None) (previous_entry is None)
or (previous_entry.status_code != code) or (previous_entry.status_code != code)
or ( or (
calculateSpeedRange(previous_entry.total_seconds, fast, moderate) calculateSpeedRange(
previous_entry.total_seconds, fast, moderate
)
!= calculateSpeedRange(total_seconds, fast, moderate) != calculateSpeedRange(total_seconds, fast, moderate)
) )
): ):
...@@ -147,7 +149,16 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id): ...@@ -147,7 +149,16 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
return previous_entry.status_id return previous_entry.status_id
def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT, elapsed_fast=ELAPSED_FAST, elapsed_moderate=ELAPSED_MODERATE): def checkHttpStatus(
db,
status_id,
url,
ip,
bot_version,
timeout=TIMEOUT,
elapsed_fast=ELAPSED_FAST,
elapsed_moderate=ELAPSED_MODERATE,
):
parsed_url = urlparse(url) parsed_url = urlparse(url)
hostname = parsed_url.hostname hostname = parsed_url.hostname
request_kw = {"timeout": timeout} request_kw = {"timeout": timeout}
......
tests/test_http.py
View file @ 916e497e
...@@ -189,7 +189,14 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -189,7 +189,14 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
assert self.db.HttpCodeChange.get().ip == ip assert self.db.HttpCodeChange.get().ip == ip
...@@ -208,11 +215,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -208,11 +215,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
try: try:
logHttpStatus( logHttpStatus(
self.db, ip, url, status_code + 1, total_seconds + 1, fast, moderate, status_id self.db,
ip,
url,
status_code + 1,
total_seconds + 1,
fast,
moderate,
status_id,
) )
except peewee.IntegrityError: except peewee.IntegrityError:
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
...@@ -229,11 +250,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -229,11 +250,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id_2,
) )
assert result_2 == result assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
...@@ -253,11 +288,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -253,11 +288,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_code_2 = status_code + 1 status_code_2 = status_code + 1
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code_2, total_seconds, fast, moderate, status_id_2 self.db,
ip,
url,
status_code_2,
total_seconds,
fast,
moderate,
status_id_2,
) )
assert result_2 != result assert result_2 != result
assert self.db.HttpCodeChange.select().count() == 2 assert self.db.HttpCodeChange.select().count() == 2
...@@ -322,19 +371,47 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -322,19 +371,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 30 moderate = 30
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds_error,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds_fast,
fast,
moderate,
status_id_2,
) )
status_id_3 = logStatus(self.db, "foo") status_id_3 = logStatus(self.db, "foo")
result_3 = logHttpStatus( result_3 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3 self.db,
ip,
url,
status_code,
total_seconds_moderate,
fast,
moderate,
status_id_3,
) )
status_id_4 = logStatus(self.db, "foo") status_id_4 = logStatus(self.db, "foo")
result_4 = logHttpStatus( result_4 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4 self.db,
ip,
url,
status_code,
total_seconds_slow,
fast,
moderate,
status_id_4,
) )
assert status_id == result assert status_id == result
assert status_id_2 == result_2 assert status_id_2 == result_2
...@@ -379,19 +456,47 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -379,19 +456,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds_error,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds_fast,
fast,
moderate,
status_id_2,
) )
status_id_3 = logStatus(self.db, "foo") status_id_3 = logStatus(self.db, "foo")
result_3 = logHttpStatus( result_3 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3 self.db,
ip,
url,
status_code,
total_seconds_moderate,
fast,
moderate,
status_id_3,
) )
status_id_4 = logStatus(self.db, "foo") status_id_4 = logStatus(self.db, "foo")
result_4 = logHttpStatus( result_4 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4 self.db,
ip,
url,
status_code,
total_seconds_slow,
fast,
moderate,
status_id_4,
) )
assert status_id == result assert status_id == result
assert status_id_2 == result_2 assert status_id_2 == result_2
...@@ -434,11 +539,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -434,11 +539,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds_2,
fast,
moderate,
status_id_2,
) )
assert result_2 == result assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
...@@ -458,11 +577,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -458,11 +577,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds_2,
fast,
moderate,
status_id_2,
) )
assert result_2 == result assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
...@@ -482,11 +615,25 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -482,11 +615,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
result = logHttpStatus( result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
status_id_2 = logStatus(self.db, "foo") status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus( result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2 self.db,
ip,
url,
status_code,
total_seconds_2,
fast,
moderate,
status_id_2,
) )
assert result_2 == result assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1 assert self.db.HttpCodeChange.select().count() == 1
...@@ -506,15 +653,45 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -506,15 +653,45 @@ class SurykatkaHttpTestCase(unittest.TestCase):
fast = 0.2 fast = 0.2
moderate = 0.5 moderate = 0.5
status_id = logStatus(self.db, "foo") status_id = logStatus(self.db, "foo")
logHttpStatus(self.db, ip, url, status_code, total_seconds, fast, moderate, status_id)
logHttpStatus( logHttpStatus(
self.db, ip_2, url, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
logHttpStatus( logHttpStatus(
self.db, ip, url_2, status_code, total_seconds, fast, moderate, status_id self.db,
ip_2,
url,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
logHttpStatus( logHttpStatus(
self.db, ip_2, url_2, status_code, total_seconds, fast, moderate, status_id self.db,
ip,
url_2,
status_code,
total_seconds,
fast,
moderate,
status_id,
)
logHttpStatus(
self.db,
ip_2,
url_2,
status_code,
total_seconds,
fast,
moderate,
status_id,
) )
assert self.db.HttpCodeChange.select().count() == 4 assert self.db.HttpCodeChange.select().count() == 4
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7