WIP: store http request elapsed time

src/surykatka/bot.py

@@ -284,6 +284,7 @@ class WebBot:
             result_dict["http_query"].append(
                 {
                     "status_code": network_change["status_code"],
+                    "total_seconds": network_change["total_seconds"],
                     "url": network_change["url"],
                     "ip": network_change["ip"],
                     "date": rfc822(network_change["status"]),

src/surykatka/db.py

@@ -20,6 +20,7 @@
 import peewee
 from playhouse.sqlite_ext import SqliteExtDatabase
 import datetime
+from playhouse.migrate import migrate, SqliteMigrator
 
 
 class LogDB:
@@ -114,6 +115,7 @@ class LogDB:
             ip = peewee.TextField()
             url = peewee.TextField()
             status_code = peewee.IntegerField()
+            total_seconds = peewee.FloatField()
 
             class Meta:
                 primary_key = peewee.CompositeKey("status", "ip", "url")
@@ -129,7 +131,7 @@ class LogDB:
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 2
+        expected_version = 3
         if db_version != expected_version:
             with self._db.transaction():
 
@@ -150,7 +152,14 @@ class LogDB:
                     # version 1 without SSL support
                     self._db.create_tables([self.SslChange])
 
-                else:
+                if (0 < db_version) and (db_version <= 2):
+                    # version 2 without the http total_seconds column
+                    migrator = SqliteMigrator(self._db)
+                    migrate(
+                        migrator.add_column("HttpCodeChange", "total_seconds")
+                    )
+
+                if db_version >= expected_version:
                     raise ValueError("Can not downgrade SQLite DB")
 
                 self._db.pragma("user_version", expected_version)

src/surykatka/http.py

@@ -78,7 +78,6 @@ def request(url, timeout=TIMEOUT, headers=None, session=requests, version=0):
     except requests.exceptions.TooManyRedirects:
         response = requests.models.Response()
         response.status_code = 520
-
     return response
 
 
@@ -106,7 +105,20 @@ def reportHttp(db, ip=None, url=None):
     return query
 
 
-def logHttpStatus(db, ip, url, code, status_id):
+def calculateSpeedRange(total_seconds):
+    # Prevent updating the DB by defining acceptable speed range
+    if total_seconds == 0:
+        # error cases
+        return "BAD"
+    elif 0.5 < total_seconds:
+        return "MODERATE"
+    elif 0.2 < total_seconds:
+        return "FAST"
+    else:
+        return "SLOW"
+
+
+def logHttpStatus(db, ip, url, code, total_seconds, status_id):
 
     with db._db.atomic():
         try:
@@ -115,9 +127,20 @@ def logHttpStatus(db, ip, url, code, status_id):
         except db.HttpCodeChange.DoesNotExist:
             previous_entry = None
 
-        if (previous_entry is None) or (previous_entry.status_code != code):
+        if (
+            (previous_entry is None)
+            or (previous_entry.status_code != code)
+            or (
+                calculateSpeedRange(previous_entry.total_seconds)
+                != calculateSpeedRange(total_seconds)
+            )
+        ):
             previous_entry = db.HttpCodeChange.create(
-                status=status_id, ip=ip, url=url, status_code=code
+                status=status_id,
+                ip=ip,
+                url=url,
+                status_code=code,
+                total_seconds=total_seconds,
             )
         return previous_entry.status_id
 
@@ -146,4 +169,11 @@ def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
     response = request(
         ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
     )
-    logHttpStatus(db, ip, url, response.status_code, status_id)
+    logHttpStatus(
+        db,
+        ip,
+        url,
+        response.status_code,
+        response.elapsed.total_seconds(),
+        status_id,
+    )