Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
surykatka
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Łukasz Nowak
surykatka
Commits
916e497e
Commit
916e497e
authored
Feb 07, 2020
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Reformatted with black
parent
24a1e7d4
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
223 additions
and
28 deletions
+223
-28
src/surykatka/bot.py
src/surykatka/bot.py
+8
-1
src/surykatka/http.py
src/surykatka/http.py
+13
-2
tests/test_http.py
tests/test_http.py
+202
-25
No files found.
src/surykatka/bot.py
View file @
916e497e
...
@@ -144,7 +144,14 @@ class WebBot:
...
@@ -144,7 +144,14 @@ class WebBot:
for
url
in
url_dict
:
for
url
in
url_dict
:
for
ip
in
url_dict
[
url
]:
for
ip
in
url_dict
[
url
]:
checkHttpStatus
(
checkHttpStatus
(
self
.
_db
,
status_id
,
url
,
ip
,
__version__
,
timeout
,
elapsed_fast
,
elapsed_moderate
self
.
_db
,
status_id
,
url
,
ip
,
__version__
,
timeout
,
elapsed_fast
,
elapsed_moderate
,
)
)
# XXX Check location header and check new url recursively
# XXX Check location header and check new url recursively
# XXX Parse HTML, fetch found link, css, js, image
# XXX Parse HTML, fetch found link, css, js, image
...
...
src/surykatka/http.py
View file @
916e497e
...
@@ -133,7 +133,9 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
...
@@ -133,7 +133,9 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
(
previous_entry
is
None
)
(
previous_entry
is
None
)
or
(
previous_entry
.
status_code
!=
code
)
or
(
previous_entry
.
status_code
!=
code
)
or
(
or
(
calculateSpeedRange
(
previous_entry
.
total_seconds
,
fast
,
moderate
)
calculateSpeedRange
(
previous_entry
.
total_seconds
,
fast
,
moderate
)
!=
calculateSpeedRange
(
total_seconds
,
fast
,
moderate
)
!=
calculateSpeedRange
(
total_seconds
,
fast
,
moderate
)
)
)
):
):
...
@@ -147,7 +149,16 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
...
@@ -147,7 +149,16 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
return
previous_entry
.
status_id
return
previous_entry
.
status_id
def
checkHttpStatus
(
db
,
status_id
,
url
,
ip
,
bot_version
,
timeout
=
TIMEOUT
,
elapsed_fast
=
ELAPSED_FAST
,
elapsed_moderate
=
ELAPSED_MODERATE
):
def
checkHttpStatus
(
db
,
status_id
,
url
,
ip
,
bot_version
,
timeout
=
TIMEOUT
,
elapsed_fast
=
ELAPSED_FAST
,
elapsed_moderate
=
ELAPSED_MODERATE
,
):
parsed_url
=
urlparse
(
url
)
parsed_url
=
urlparse
(
url
)
hostname
=
parsed_url
.
hostname
hostname
=
parsed_url
.
hostname
request_kw
=
{
"timeout"
:
timeout
}
request_kw
=
{
"timeout"
:
timeout
}
...
...
tests/test_http.py
View file @
916e497e
...
@@ -189,7 +189,14 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -189,7 +189,14 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
get
().
ip
==
ip
assert
self
.
db
.
HttpCodeChange
.
get
().
ip
==
ip
...
@@ -208,11 +215,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -208,11 +215,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
try
:
try
:
logHttpStatus
(
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
+
1
,
total_seconds
+
1
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
+
1
,
total_seconds
+
1
,
fast
,
moderate
,
status_id
,
)
)
except
peewee
.
IntegrityError
:
except
peewee
.
IntegrityError
:
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
...
@@ -229,11 +250,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -229,11 +250,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id_2
,
)
)
assert
result_2
==
result
assert
result_2
==
result
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
...
@@ -253,11 +288,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -253,11 +288,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_code_2
=
status_code
+
1
status_code_2
=
status_code
+
1
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code_2
,
total_seconds
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code_2
,
total_seconds
,
fast
,
moderate
,
status_id_2
,
)
)
assert
result_2
!=
result
assert
result_2
!=
result
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
2
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
2
...
@@ -322,19 +371,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -322,19 +371,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
30
moderate
=
30
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_error
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_error
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_fast
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_fast
,
fast
,
moderate
,
status_id_2
,
)
)
status_id_3
=
logStatus
(
self
.
db
,
"foo"
)
status_id_3
=
logStatus
(
self
.
db
,
"foo"
)
result_3
=
logHttpStatus
(
result_3
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_moderate
,
fast
,
moderate
,
status_id_3
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_moderate
,
fast
,
moderate
,
status_id_3
,
)
)
status_id_4
=
logStatus
(
self
.
db
,
"foo"
)
status_id_4
=
logStatus
(
self
.
db
,
"foo"
)
result_4
=
logHttpStatus
(
result_4
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_slow
,
fast
,
moderate
,
status_id_4
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_slow
,
fast
,
moderate
,
status_id_4
,
)
)
assert
status_id
==
result
assert
status_id
==
result
assert
status_id_2
==
result_2
assert
status_id_2
==
result_2
...
@@ -379,19 +456,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -379,19 +456,47 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_error
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_error
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_fast
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_fast
,
fast
,
moderate
,
status_id_2
,
)
)
status_id_3
=
logStatus
(
self
.
db
,
"foo"
)
status_id_3
=
logStatus
(
self
.
db
,
"foo"
)
result_3
=
logHttpStatus
(
result_3
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_moderate
,
fast
,
moderate
,
status_id_3
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_moderate
,
fast
,
moderate
,
status_id_3
,
)
)
status_id_4
=
logStatus
(
self
.
db
,
"foo"
)
status_id_4
=
logStatus
(
self
.
db
,
"foo"
)
result_4
=
logHttpStatus
(
result_4
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_slow
,
fast
,
moderate
,
status_id_4
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_slow
,
fast
,
moderate
,
status_id_4
,
)
)
assert
status_id
==
result
assert
status_id
==
result
assert
status_id_2
==
result_2
assert
status_id_2
==
result_2
...
@@ -434,11 +539,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -434,11 +539,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
,
)
)
assert
result_2
==
result
assert
result_2
==
result
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
...
@@ -458,11 +577,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -458,11 +577,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
,
)
)
assert
result_2
==
result
assert
result_2
==
result
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
...
@@ -482,11 +615,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -482,11 +615,25 @@ class SurykatkaHttpTestCase(unittest.TestCase):
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
result
=
logHttpStatus
(
result
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
status_id_2
=
logStatus
(
self
.
db
,
"foo"
)
result_2
=
logHttpStatus
(
result_2
=
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
self
.
db
,
ip
,
url
,
status_code
,
total_seconds_2
,
fast
,
moderate
,
status_id_2
,
)
)
assert
result_2
==
result
assert
result_2
==
result
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
1
...
@@ -506,15 +653,45 @@ class SurykatkaHttpTestCase(unittest.TestCase):
...
@@ -506,15 +653,45 @@ class SurykatkaHttpTestCase(unittest.TestCase):
fast
=
0.2
fast
=
0.2
moderate
=
0.5
moderate
=
0.5
status_id
=
logStatus
(
self
.
db
,
"foo"
)
status_id
=
logStatus
(
self
.
db
,
"foo"
)
logHttpStatus
(
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
)
logHttpStatus
(
logHttpStatus
(
self
.
db
,
ip_2
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
logHttpStatus
(
logHttpStatus
(
self
.
db
,
ip
,
url_2
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip_2
,
url
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
logHttpStatus
(
logHttpStatus
(
self
.
db
,
ip_2
,
url_2
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
self
.
db
,
ip
,
url_2
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
logHttpStatus
(
self
.
db
,
ip_2
,
url_2
,
status_code
,
total_seconds
,
fast
,
moderate
,
status_id
,
)
)
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
4
assert
self
.
db
.
HttpCodeChange
.
select
().
count
()
==
4
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment