Commit 92bdae55 authored by Romain Courteaud's avatar Romain Courteaud

Add various improvements: pack, warning, speed, url suffix, missing data

See merge request nexedi/surykatka!12
parents a25dd45a 8ff446bc
...@@ -49,6 +49,9 @@ setup( ...@@ -49,6 +49,9 @@ setup(
"dev": ["pytest", "black", "pyflakes", "mock", "httpretty"] "dev": ["pytest", "black", "pyflakes", "mock", "httpretty"]
}, },
entry_points={ entry_points={
"console_scripts": ["surykatka=surykatka.cli:runSurykatka "] "console_scripts": [
"surykatka=surykatka.cli:runSurykatka ",
"surykatkastat=surykatka.cli:runStats ",
]
}, },
) )
...@@ -26,17 +26,29 @@ from .dns import ( ...@@ -26,17 +26,29 @@ from .dns import (
expandDomainList, expandDomainList,
getDomainIpDict, getDomainIpDict,
reportDnsQuery, reportDnsQuery,
packDns,
) )
from .http import getRootUrl, getUrlHostname, checkHttpStatus, reportHttp from .http import (
from .network import isTcpPortOpen, reportNetwork getRootUrl,
getUrlHostname,
checkHttpStatus,
reportHttp,
packHttp,
)
from .network import isTcpPortOpen, reportNetwork, packNetwork
import json import json
import email.utils import email.utils
from collections import OrderedDict from collections import OrderedDict
from .ssl import hasValidSSLCertificate, reportSslCertificate from .ssl import (
hasValidSSLCertificate,
reportSslCertificate,
packSslCertificate,
)
import datetime import datetime
from email.utils import parsedate_to_datetime
__version__ = "0.5.0" __version__ = "0.6.0"
class BotError(Exception): class BotError(Exception):
...@@ -47,6 +59,94 @@ def rfc822(date): ...@@ -47,6 +59,94 @@ def rfc822(date):
return email.utils.format_datetime(date) return email.utils.format_datetime(date)
def filterWarningStatus(status_dict, interval, not_critical_url_list):
now = datetime.datetime.utcnow()
if interval < 60:
interval = 60
for i in range(len(status_dict["bot_status"]) - 1, -1, -1):
status_date = parsedate_to_datetime(
status_dict["bot_status"][i]["date"]
)
if (now - status_date).total_seconds() < (2 * interval):
# Skip the bot status if it was recently triggerer
del status_dict["bot_status"][i]
if not status_dict["bot_status"]:
del status_dict["bot_status"]
for i in range(len(status_dict["dns_server"]) - 1, -1, -1):
state = status_dict["dns_server"][i]["state"]
if state == "open":
del status_dict["dns_server"][i]
if not status_dict["dns_server"]:
del status_dict["dns_server"]
for i in range(len(status_dict["dns_query"]) - 1, -1, -1):
state = status_dict["dns_query"][i]["response"]
if state != "":
del status_dict["dns_query"][i]
if not status_dict["dns_query"]:
del status_dict["dns_query"]
if not status_dict["missing_data"]:
del status_dict["missing_data"]
for i in range(len(status_dict["http_server"]) - 1, -1, -1):
state = status_dict["http_server"][i]["state"]
# Skip if all domains lead to not critical urls
prefix = ""
if status_dict["http_server"][i]["port"] == 80:
prefix = "http://"
elif status_dict["http_server"][i]["port"] == 443:
prefix = "https://"
domain_list = status_dict["http_server"][i]["domain"].split(", ")
domain_list = [
x
for x in domain_list
if "%s%s" % (prefix, x) not in not_critical_url_list
]
if (state == "open") or (not domain_list):
del status_dict["http_server"][i]
if not status_dict["http_server"]:
del status_dict["http_server"]
for i in range(len(status_dict["ssl_certificate"]) - 1, -1, -1):
not_after = status_dict["ssl_certificate"][i]["not_after"]
if (
(not_after is not None)
and (
(60 * 60 * 24 * 14)
< (parsedate_to_datetime(not_after) - now).total_seconds()
)
) or (
("https://%s" % status_dict["ssl_certificate"][i]["hostname"])
in not_critical_url_list
):
# Warn 2 weeks before expiration
# Skip if we check only the http url
del status_dict["ssl_certificate"][i]
else:
# Drop columns with too much info
del status_dict["ssl_certificate"][i]["not_before"]
del status_dict["ssl_certificate"][i]["issuer"]
del status_dict["ssl_certificate"][i]["sha1_fingerprint"]
del status_dict["ssl_certificate"][i]["subject"]
if not status_dict["ssl_certificate"]:
del status_dict["ssl_certificate"]
for i in range(len(status_dict["http_query"]) - 1, -1, -1):
http_code = status_dict["http_query"][i]["status_code"]
if (http_code != 404) and (http_code < 500):
del status_dict["http_query"][i]
elif status_dict["http_query"][i]["url"] in not_critical_url_list:
del status_dict["http_query"][i]
else:
# Drop columns with too much info
del status_dict["http_query"][i]["http_header_dict"]
del status_dict["http_query"][i]["total_seconds"]
if not status_dict["http_query"]:
del status_dict["http_query"]
class WebBot: class WebBot:
def __init__(self, **kw): def __init__(self, **kw):
self.config_kw = kw self.config_kw = kw
...@@ -81,6 +181,22 @@ class WebBot: ...@@ -81,6 +181,22 @@ class WebBot:
public_suffix_list=self.config["PUBLIC_SUFFIX"].split(), public_suffix_list=self.config["PUBLIC_SUFFIX"].split(),
) )
def calculateNotCriticalUrlList(self):
domain_list = self.config["DOMAIN"].split()
url_list = self.config["URL"].split()
not_critical_url_list = []
for url in url_list:
hostname = getUrlHostname(url)
if hostname is not None:
if hostname not in domain_list:
# Domain not explicitely checked
# Skip both root url
for protocol in ("http", "https"):
not_critical_url = "%s://%s" % (protocol, hostname)
if not_critical_url not in url_list:
not_critical_url_list.append(not_critical_url)
return not_critical_url_list
def iterateLoop(self): def iterateLoop(self):
status_id = logStatus(self._db, "loop") status_id = logStatus(self._db, "loop")
...@@ -347,12 +463,20 @@ class WebBot: ...@@ -347,12 +463,20 @@ class WebBot:
self._running = True self._running = True
try: try:
while self._running: while self._running:
previous_time = datetime.datetime.utcnow()
self.iterateLoop() self.iterateLoop()
next_time = datetime.datetime.utcnow()
interval = int(self.config.get("INTERVAL")) interval = int(self.config.get("INTERVAL"))
if interval < 0: if interval < 0:
self.stop() self.stop()
else: else:
time.sleep(interval) time.sleep(
max(
0,
interval
- (next_time - previous_time).total_seconds(),
)
)
except KeyboardInterrupt: except KeyboardInterrupt:
self.stop() self.stop()
except: except:
...@@ -360,22 +484,36 @@ class WebBot: ...@@ -360,22 +484,36 @@ class WebBot:
logStatus(self._db, "error") logStatus(self._db, "error")
raise raise
def pack(self):
logStatus(self._db, "packing")
packDns(self._db)
packHttp(self._db)
packNetwork(self._db)
packSslCertificate(self._db)
self._db.vacuum()
logStatus(self._db, "packed")
def run(self, mode): def run(self, mode):
status_dict = None status_dict = None
if mode not in ["crawl", "status"]: if mode not in ["crawl", "pack", "status", "warning"]:
raise NotImplementedError("Unexpected mode: %s" % mode) raise NotImplementedError("Unexpected mode: %s" % mode)
if self.config["SQLITE"] == ":memory:": if self.config["SQLITE"] == ":memory:":
# Crawl/report are mandatory when using memory # Crawl/report are mandatory when using memory
if mode == "warning":
mode = "wallwarning"
else:
mode = "all" mode = "all"
self.initDB() self.initDB()
try: try:
if mode in ["crawl", "all"]: if mode in ["crawl", "wallwarning", "all"]:
self.crawl() self.crawl()
if mode in ["status", "all"]: if mode in ["status", "all", "wallwarning", "warning"]:
status_dict = self.status() status_dict = self.status()
if mode == "pack":
self.pack()
except: except:
self.closeDB() self.closeDB()
raise raise
...@@ -383,6 +521,12 @@ class WebBot: ...@@ -383,6 +521,12 @@ class WebBot:
self.closeDB() self.closeDB()
if status_dict is not None: if status_dict is not None:
if mode in ("wallwarning", "warning"):
filterWarningStatus(
status_dict,
int(self.config.get("INTERVAL")),
self.calculateNotCriticalUrlList(),
)
if self.config["FORMAT"] == "json": if self.config["FORMAT"] == "json":
print(json.dumps(status_dict)) print(json.dumps(status_dict))
else: else:
......
...@@ -29,7 +29,7 @@ from .bot import create_bot ...@@ -29,7 +29,7 @@ from .bot import create_bot
help="The bot operation mode to run.", help="The bot operation mode to run.",
show_default=True, show_default=True,
default="status", default="status",
type=click.Choice(["crawl", "status"]), type=click.Choice(["crawl", "pack", "status", "warning"]),
) )
@click.option( @click.option(
"--sqlite", "-s", help="The path of the sqlite DB. (default: :memory:)" "--sqlite", "-s", help="The path of the sqlite DB. (default: :memory:)"
...@@ -55,6 +55,9 @@ from .bot import create_bot ...@@ -55,6 +55,9 @@ from .bot import create_bot
default="plain", default="plain",
show_default=True, show_default=True,
) )
@click.option(
"--profile", help="Profiler data path", type=click.Path(exists=False)
)
def runSurykatka( def runSurykatka(
run, run,
sqlite, sqlite,
...@@ -65,6 +68,7 @@ def runSurykatka( ...@@ -65,6 +68,7 @@ def runSurykatka(
configuration, configuration,
reload, reload,
output, output,
profile,
): ):
mapping = {} mapping = {}
...@@ -85,7 +89,28 @@ def runSurykatka( ...@@ -85,7 +89,28 @@ def runSurykatka(
mapping["RELOAD"] = str(reload) mapping["RELOAD"] = str(reload)
mapping["FORMAT"] = output mapping["FORMAT"] = output
bot = create_bot(cfgfile=configuration, mapping=mapping) bot = create_bot(cfgfile=configuration, mapping=mapping)
if profile is None:
return bot.run(run) return bot.run(run)
else:
import cProfile
return cProfile.runctx(
"bot.run(run)", globals(), locals(), filename=profile
)
@click.command(short_help="Stats profiler bot data.")
@click.option("--stats", type=click.Choice(["cumul", "time"]))
@click.argument("profile", type=click.Path(exists=True, dir_okay=False))
def runStats(stats, profile):
click.echo("Profile bot execution")
import pstats
profile_stats = pstats.Stats(profile)
if stats == "time":
profile_stats.sort_stats("time", "calls").print_stats(30)
else:
profile_stats.sort_stats("cumulative").print_stats(30)
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -26,7 +26,15 @@ from playhouse.migrate import migrate, SqliteMigrator ...@@ -26,7 +26,15 @@ from playhouse.migrate import migrate, SqliteMigrator
class LogDB: class LogDB:
def __init__(self, sqlite_path): def __init__(self, sqlite_path):
self._db = SqliteExtDatabase( self._db = SqliteExtDatabase(
sqlite_path, pragmas=(("journal_mode", "WAL"), ("foreign_keys", 1)) sqlite_path,
pragmas=(
("journal_mode", "WAL"),
("foreign_keys", 1),
("wal_autocheckpoint", 5),
("temp_store", "MEMORY"),
("synchronous", "NORMAL"),
("mmap_size", 30000000000),
),
) )
self._db.connect() self._db.connect()
...@@ -228,3 +236,6 @@ class LogDB: ...@@ -228,3 +236,6 @@ class LogDB:
def close(self): def close(self):
self._db.close() self._db.close()
def vacuum(self):
self._db.execute_sql("VACUUM", [])
...@@ -52,6 +52,18 @@ def reportDnsQuery(db, resolver_ip=None, domain=None, rdtype=None): ...@@ -52,6 +52,18 @@ def reportDnsQuery(db, resolver_ip=None, domain=None, rdtype=None):
return query return query
def packDns(db):
with db._db.atomic():
result = [x for x in reportDnsQuery(db)]
for dns_change in result:
db.DnsChange.delete().where(
db.DnsChange.status_id != dns_change.status_id,
db.DnsChange.resolver_ip == dns_change.resolver_ip,
db.DnsChange.domain == dns_change.domain,
db.DnsChange.rdtype == dns_change.rdtype,
).execute()
def logDnsQuery(db, status_id, resolver_ip, domain_text, rdtype, answer_list): def logDnsQuery(db, status_id, resolver_ip, domain_text, rdtype, answer_list):
answer_list.sort() answer_list.sort()
response = ", ".join(answer_list) response = ", ".join(answer_list)
......
...@@ -74,7 +74,10 @@ def request(url, timeout=TIMEOUT, headers=None, session=requests, version=0): ...@@ -74,7 +74,10 @@ def request(url, timeout=TIMEOUT, headers=None, session=requests, version=0):
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
response = requests.models.Response() response = requests.models.Response()
response.status_code = 523 response.status_code = 523
except requests.exceptions.Timeout: except (
requests.exceptions.Timeout,
requests.exceptions.ChunkedEncodingError,
):
response = requests.models.Response() response = requests.models.Response()
response.status_code = 524 response.status_code = 524
except requests.exceptions.TooManyRedirects: except requests.exceptions.TooManyRedirects:
...@@ -107,6 +110,17 @@ def reportHttp(db, ip=None, url=None): ...@@ -107,6 +110,17 @@ def reportHttp(db, ip=None, url=None):
return query return query
def packHttp(db):
with db._db.atomic():
result = [x for x in reportHttp(db)]
for http_change in result:
db.HttpCodeChange.delete().where(
db.HttpCodeChange.status_id != http_change.status_id,
db.HttpCodeChange.url == http_change.url,
db.HttpCodeChange.ip == http_change.ip,
).execute()
def calculateSpeedRange(total_seconds, fast, moderate): def calculateSpeedRange(total_seconds, fast, moderate):
# Prevent updating the DB by defining acceptable speed range # Prevent updating the DB by defining acceptable speed range
if total_seconds == 0: if total_seconds == 0:
......
...@@ -56,6 +56,18 @@ def reportNetwork(db, ip=None, transport=None, port=None): ...@@ -56,6 +56,18 @@ def reportNetwork(db, ip=None, transport=None, port=None):
return query return query
def packNetwork(db):
with db._db.atomic():
result = [x for x in reportNetwork(db)]
for network_change in result:
db.NetworkChange.delete().where(
db.NetworkChange.status_id != network_change.status_id,
db.NetworkChange.transport == network_change.transport,
db.NetworkChange.port == network_change.port,
db.NetworkChange.ip == network_change.ip,
).execute()
def logNetwork(db, ip, transport, port, state, status_id): def logNetwork(db, ip, transport, port, state, status_id):
with db._db.atomic(): with db._db.atomic():
......
...@@ -53,6 +53,18 @@ def reportSslCertificate(db, ip=None, port=None, hostname=None): ...@@ -53,6 +53,18 @@ def reportSslCertificate(db, ip=None, port=None, hostname=None):
return query return query
def packSslCertificate(db):
with db._db.atomic():
result = [x for x in reportSslCertificate(db)]
for ssl_change in result:
db.SslChange.delete().where(
db.SslChange.status_id != ssl_change.status_id,
db.SslChange.hostname == ssl_change.hostname,
db.SslChange.port == ssl_change.port,
db.SslChange.ip == ssl_change.ip,
).execute()
def logSslCertificate( def logSslCertificate(
db, db,
ip, ip,
......
...@@ -679,6 +679,7 @@ class SurykatkaBotStatusTestCase(unittest.TestCase): ...@@ -679,6 +679,7 @@ class SurykatkaBotStatusTestCase(unittest.TestCase):
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(SurykatkaBotTestCase)) suite.addTest(unittest.makeSuite(SurykatkaBotTestCase))
suite.addTest(unittest.makeSuite(SurykatkaBotStatusTestCase))
return suite return suite
......
...@@ -23,6 +23,7 @@ import peewee ...@@ -23,6 +23,7 @@ import peewee
import surykatka.dns import surykatka.dns
from surykatka.dns import ( from surykatka.dns import (
expandDomainList, expandDomainList,
packDns,
logDnsQuery, logDnsQuery,
buildResolver, buildResolver,
queryDNS, queryDNS,
...@@ -586,6 +587,78 @@ class SurykatkaDNSTestCase(unittest.TestCase): ...@@ -586,6 +587,78 @@ class SurykatkaDNSTestCase(unittest.TestCase):
assert self.db.DnsChange.select().count() == 2 assert self.db.DnsChange.select().count() == 2
assert self.db.NetworkChange.select().count() == 0 assert self.db.NetworkChange.select().count() == 0
################################################
# packDns
################################################
def test_packDns_oldLog(self):
domain = "http://example.org"
resolver_ip = "127.0.0.1"
rdtype = "foo"
answer_list = ["4.3.2.1", "1.2.3.4"]
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip, domain, rdtype, answer_list
)
answer_list_2 = ["4.3.2.1", "1.2.3.4", "0.0.0.0"]
status_id_2 = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id_2, resolver_ip, domain, rdtype, answer_list_2
)
result = packDns(self.db)
assert self.db.DnsChange.select().count() == 1
assert self.db.DnsChange.get().resolver_ip == resolver_ip
assert self.db.DnsChange.get().domain == domain
assert self.db.DnsChange.get().rdtype == rdtype
assert self.db.DnsChange.get().response == "0.0.0.0, 1.2.3.4, 4.3.2.1"
assert self.db.DnsChange.get().status_id == status_id_2
assert result == None
def test_packDns_keepDifferentUrl(self):
domain = "http://example.org"
domain_2 = domain + "."
resolver_ip = "127.0.0.1"
resolver_ip_2 = resolver_ip + "1"
rdtype = "foo"
rdtype_2 = rdtype + "bar"
answer_list = ["4.3.2.1", "1.2.3.4"]
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip, domain, rdtype, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip_2, domain, rdtype, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip, domain_2, rdtype, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip, domain, rdtype_2, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip_2, domain_2, rdtype, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip_2, domain, rdtype_2, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip, domain_2, rdtype_2, answer_list
)
status_id = logStatus(self.db, "foo")
logDnsQuery(
self.db, status_id, resolver_ip_2, domain_2, rdtype_2, answer_list
)
result = packDns(self.db)
assert self.db.DnsChange.select().count() == 8
assert result == None
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
...@@ -27,6 +27,7 @@ from surykatka.http import ( ...@@ -27,6 +27,7 @@ from surykatka.http import (
request, request,
logHttpStatus, logHttpStatus,
checkHttpStatus, checkHttpStatus,
packHttp,
) )
from surykatka.status import logStatus from surykatka.status import logStatus
import httpretty import httpretty
...@@ -151,6 +152,19 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -151,6 +152,19 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert mock_request.call_count == 1 assert mock_request.call_count == 1
assert response.status_code == 524, response.status_code assert response.status_code == 524, response.status_code
def test_request_ChunkedEncodingError(self):
url_to_proxy = "http://example.org/"
httpretty.register_uri(httpretty.GET, url_to_proxy)
with mock.patch("surykatka.http.requests.request") as mock_request:
def sideEffect(*args, **kw):
raise surykatka.http.requests.exceptions.ChunkedEncodingError()
mock_request.side_effect = sideEffect
response = request(url_to_proxy)
assert mock_request.call_count == 1
assert response.status_code == 524, response.status_code
def test_request_tooManyRedirect(self): def test_request_tooManyRedirect(self):
url_to_proxy = "http://example.org/" url_to_proxy = "http://example.org/"
httpretty.register_uri(httpretty.GET, url_to_proxy) httpretty.register_uri(httpretty.GET, url_to_proxy)
...@@ -1037,6 +1051,113 @@ class SurykatkaHttpTestCase(unittest.TestCase): ...@@ -1037,6 +1051,113 @@ class SurykatkaHttpTestCase(unittest.TestCase):
) )
assert self.db.HttpCodeChange.get().status_id == status_id assert self.db.HttpCodeChange.get().status_id == status_id
################################################
# packHttp
################################################
def test_packHttp_dropOldLog(self):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
status_code_2 = status_code + 1
status_id = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
status_id_2 = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip,
url,
status_code_2,
http_header_dict,
total_seconds,
fast,
moderate,
status_id_2,
)
result = packHttp(self.db)
assert self.db.HttpCodeChange.select().count() == 1
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code_2
assert self.db.HttpCodeChange.get().status_id == status_id_2
assert result == None
def test_packHttp_keepDifferentUrl(self):
ip = "127.0.0.1"
ip_2 = ip + "2"
url = "http://example.org"
url_2 = url + "2"
total_seconds = 0.1
status_code = 200
http_header_dict = {"a": "b"}
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
status_id = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip_2,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
status_id = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip,
url_2,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
status_id = logStatus(self.db, "foo")
logHttpStatus(
self.db,
ip_2,
url_2,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
packHttp(self.db)
assert self.db.HttpCodeChange.select().count() == 4
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
import unittest import unittest
from surykatka.db import LogDB from surykatka.db import LogDB
import surykatka.network import surykatka.network
from surykatka.network import logNetwork, isTcpPortOpen from surykatka.network import logNetwork, isTcpPortOpen, packNetwork
from surykatka.status import logStatus from surykatka.status import logStatus
import mock import mock
import peewee import peewee
...@@ -422,6 +422,59 @@ class SurykatkaNetworkTestCase(unittest.TestCase): ...@@ -422,6 +422,59 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
assert mock_socket.return_value.close.call_count == 1 assert mock_socket.return_value.close.call_count == 1
################################################
# packNetwork
################################################
def test_packNetwork_oldLog(self):
ip = "127.0.0.1"
port = 1234
transport = "foobar"
state = "bar"
state_2 = "bar2"
status_id = logStatus(self.db, "foo")
status_id_2 = logStatus(self.db, "foo")
logNetwork(self.db, ip, transport, port, state, status_id)
logNetwork(self.db, ip, transport, port, state_2, status_id_2)
result = packNetwork(self.db)
assert self.db.NetworkChange.select().count() == 1
assert self.db.NetworkChange.get().ip == ip
assert self.db.NetworkChange.get().port == port
assert self.db.NetworkChange.get().transport == transport
assert self.db.NetworkChange.get().state == state_2
assert self.db.NetworkChange.get().status_id == status_id_2
assert result == None
def test_packNetwork_keepDifferentUrl(self):
ip = "127.0.0.1"
ip_2 = ip + "2"
port = 1234
port_2 = port + 1
transport = "foobar"
transport_2 = transport + "."
state = "bar"
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip, transport, port, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip_2, transport, port, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip, transport_2, port, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip, transport, port_2, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip_2, transport_2, port, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip_2, transport, port_2, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip, transport_2, port_2, state, status_id)
status_id = logStatus(self.db, "foo")
logNetwork(self.db, ip_2, transport_2, port_2, state, status_id)
result = packNetwork(self.db)
assert self.db.NetworkChange.select().count() == 8
assert result == None
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
...@@ -20,7 +20,11 @@ ...@@ -20,7 +20,11 @@
import unittest import unittest
from surykatka.db import LogDB from surykatka.db import LogDB
import surykatka.ssl import surykatka.ssl
from surykatka.ssl import logSslCertificate, hasValidSSLCertificate from surykatka.ssl import (
logSslCertificate,
hasValidSSLCertificate,
packSslCertificate,
)
from surykatka.status import logStatus from surykatka.status import logStatus
import mock import mock
import peewee import peewee
...@@ -719,6 +723,91 @@ class SurykatkaSslTestCase(unittest.TestCase): ...@@ -719,6 +723,91 @@ class SurykatkaSslTestCase(unittest.TestCase):
== 0 == 0
) )
################################################
# packSslCertificate
################################################
def test_packSslCertificate_differentState(self):
ip = "127.0.0.1"
port = 1234
hostname = "example.org"
sha1_fingerprint = "asdfghj"
not_before = datetime.datetime.utcnow()
not_after = datetime.datetime.utcnow()
subject = "foosubject"
issuer = "barissuer"
status_id = logStatus(self.db, "foo")
logSslCertificate(
self.db,
ip,
port,
hostname,
sha1_fingerprint,
not_before,
not_after,
subject,
issuer,
status_id,
)
status_id_2 = logStatus(self.db, "foo")
sha1_fingerprint_2 = sha1_fingerprint + "."
logSslCertificate(
self.db,
ip,
port,
hostname,
sha1_fingerprint_2,
not_before,
not_after,
subject,
issuer,
status_id_2,
)
result = packSslCertificate(self.db)
assert self.db.SslChange.select().count() == 1
assert self.db.SslChange.get().ip == ip
assert self.db.SslChange.get().port == port
assert self.db.SslChange.get().hostname == hostname
assert self.db.SslChange.get().sha1_fingerprint == sha1_fingerprint_2
assert self.db.SslChange.get().status_id == status_id_2
assert result == None
def test_packSslCertificate_keepDifferentKeys(self):
ip = "127.0.0.1"
ip_2 = ip + "2"
port = 1234
port_2 = port + 1
hostname = "example.org"
hostname_2 = hostname + "."
status_id = logStatus(self.db, "foo")
sha1_fingerprint = "asdfghj"
not_before = datetime.datetime.utcnow()
not_after = datetime.datetime.utcnow()
subject = "foosubject"
issuer = "barissuer"
args = [
sha1_fingerprint,
not_before,
not_after,
subject,
issuer,
status_id,
]
logSslCertificate(self.db, ip, port, hostname, *args)
logSslCertificate(self.db, ip_2, port, hostname, *args)
logSslCertificate(self.db, ip, port_2, hostname, *args)
logSslCertificate(self.db, ip, port, hostname_2, *args)
logSslCertificate(self.db, ip_2, port_2, hostname, *args)
logSslCertificate(self.db, ip_2, port, hostname_2, *args)
logSslCertificate(self.db, ip, port_2, hostname_2, *args)
logSslCertificate(self.db, ip_2, port_2, hostname_2, *args)
assert self.db.SslChange.select().count() == 8
def suite(): def suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment