Use timestamp as status primary key

surykatka_bot.py

@@ -165,7 +165,7 @@ class WebBot:
                 {
                     "ip": network_change["ip"],
                     "state": network_change["state"],
-                    "date": rfc822(network_change["timestamp"]),
+                    "date": rfc822(network_change["status"]),
                 }
             )
 
@@ -184,7 +184,7 @@ class WebBot:
                 {
                     "domain": dns_change["domain"],
                     "resolver_ip": dns_change["resolver_ip"],
-                    "date": rfc822(dns_change["timestamp"]),
+                    "date": rfc822(dns_change["status"]),
                     "response": dns_change["response"],
                 }
             )
@@ -212,7 +212,7 @@ class WebBot:
                     "ip": network_change["ip"],
                     "state": network_change["state"],
                     "port": network_change["port"],
-                    "date": rfc822(network_change["timestamp"]),
+                    "date": rfc822(network_change["status"]),
                     "domain": ", ".join(server_ip_dict[network_change["ip"]]),
                 }
             )
@@ -246,7 +246,7 @@ class WebBot:
                     "status_code": network_change["status_code"],
                     "url": network_change["url"],
                     "ip": network_change["ip"],
-                    "date": rfc822(network_change["timestamp"]),
+                    "date": rfc822(network_change["status"]),
                 }
             )
 

surykatka_db.py

@@ -37,12 +37,11 @@ class LogDB:
         # This store the start, stop, loop time of the bot
         # All other tables point to it to be able to group some info
         class Status(BaseModel):
-            # Ensure the id is always incremented
-            id = peewee.AutoField()
             text = peewee.TextField()
             timestamp = peewee.TimestampField(
+                primary_key=True,
                 # Store millisecond resolution
-                resolution=3,
+                resolution=6,
                 # date is in UTC
                 utc=True,
                 default=datetime.datetime.now,

surykatka_dns.py

@@ -26,44 +26,12 @@ TIMEOUT = 2
 
 
 def reportDnsQuery(db, resolver_ip=None, domain=None, rdtype=None):
-    # http://charlesleifer.com/blog/techniques-for-querying-lists-of-objects-and-determining-the-top-related-item/
-    DnsChangeAlias = db.DnsChange.alias()
-
-    subquery = (
-        DnsChangeAlias.select(
-            DnsChangeAlias.resolver_ip,
-            DnsChangeAlias.domain,
-            DnsChangeAlias.rdtype,
-            fn.MAX(DnsChangeAlias.status_id).alias("max_status_id"),
-        )
-        .group_by(
-            DnsChangeAlias.resolver_ip,
-            DnsChangeAlias.domain,
-            DnsChangeAlias.rdtype,
-        )
-        .alias("dns_change_max_subquery")
-    )
-
-    StatusAlias = db.Status.alias()
-
     query = (
-        db.DnsChange.select(db.DnsChange, StatusAlias)
-        .join(StatusAlias)
-        .switch(db.DnsChange)
-        .join(
-            subquery,
-            on=(
-                (db.DnsChange.status_id == subquery.c.max_status_id)
-                & (db.DnsChange.resolver_ip == subquery.c.resolver_ip)
-                & (db.DnsChange.domain == subquery.c.domain)
-                & (db.DnsChange.rdtype == subquery.c.rdtype)
-            ),
-        )
-        .order_by(
-            db.DnsChange.domain.asc(),
-            db.DnsChange.rdtype.asc(),
-            db.DnsChange.resolver_ip.asc(),
+        db.DnsChange.select(db.DnsChange)
+        .group_by(
+            db.DnsChange.resolver_ip, db.DnsChange.domain, db.DnsChange.rdtype
         )
+        .having(db.DnsChange.status_id == fn.MAX(db.DnsChange.status_id))
     )
 
     if resolver_ip is not None:

surykatka_http.py

@@ -83,34 +83,12 @@ def request(url, timeout=TIMEOUT, headers=None, session=requests, version=0):
 
 
 def reportHttp(db, ip=None, url=None):
-    # http://charlesleifer.com/blog/techniques-for-querying-lists-of-objects-and-determining-the-top-related-item/
-    HttpCodeChangeAlias = db.HttpCodeChange.alias()
-
-    subquery = (
-        HttpCodeChangeAlias.select(
-            HttpCodeChangeAlias.ip,
-            HttpCodeChangeAlias.url,
-            fn.MAX(HttpCodeChangeAlias.status_id).alias("max_status_id"),
-        )
-        .group_by(HttpCodeChangeAlias.ip, HttpCodeChangeAlias.url)
-        .alias("http_change_max_subquery")
-    )
-
-    StatusAlias = db.Status.alias()
-
     query = (
-        db.HttpCodeChange.select(db.HttpCodeChange, StatusAlias)
-        .join(StatusAlias)
-        .switch(db.HttpCodeChange)
-        .join(
-            subquery,
-            on=(
-                (db.HttpCodeChange.status_id == subquery.c.max_status_id)
-                & (db.HttpCodeChange.ip == subquery.c.ip)
-                & (db.HttpCodeChange.url == subquery.c.url)
-            ),
+        db.HttpCodeChange.select(db.HttpCodeChange)
+        .group_by(db.HttpCodeChange.ip, db.HttpCodeChange.url)
+        .having(
+            db.HttpCodeChange.status_id == fn.MAX(db.HttpCodeChange.status_id)
         )
-        .order_by(db.HttpCodeChange.url.asc(), db.HttpCodeChange.ip.asc())
     )
 
     if ip is not None:

surykatka_network.py

@@ -26,43 +26,15 @@ TIMEOUT = 2
 
 
 def reportNetwork(db, ip=None, transport=None, port=None):
-    # http://charlesleifer.com/blog/techniques-for-querying-lists-of-objects-and-determining-the-top-related-item/
-    NetworkChangeAlias = db.NetworkChange.alias()
-
-    subquery = (
-        NetworkChangeAlias.select(
-            NetworkChangeAlias.ip,
-            NetworkChangeAlias.transport,
-            NetworkChangeAlias.port,
-            fn.MAX(NetworkChangeAlias.status_id).alias("max_status_id"),
-        )
-        .group_by(
-            NetworkChangeAlias.ip,
-            NetworkChangeAlias.transport,
-            NetworkChangeAlias.port,
-        )
-        .alias("network_change_max_subquery")
-    )
-
-    StatusAlias = db.Status.alias()
-
     query = (
-        db.NetworkChange.select(db.NetworkChange, StatusAlias)
-        .join(StatusAlias)
-        .switch(db.NetworkChange)
-        .join(
-            subquery,
-            on=(
-                (db.NetworkChange.status_id == subquery.c.max_status_id)
-                & (db.NetworkChange.ip == subquery.c.ip)
-                & (db.NetworkChange.transport == subquery.c.transport)
-                & (db.NetworkChange.port == subquery.c.port)
-            ),
+        db.NetworkChange.select(db.NetworkChange)
+        .group_by(
+            db.NetworkChange.ip,
+            db.NetworkChange.transport,
+            db.NetworkChange.port,
         )
-        .order_by(
-            db.NetworkChange.ip.asc(),
-            db.NetworkChange.port.asc(),
-            db.NetworkChange.transport.asc(),
+        .having(
+            db.NetworkChange.status_id == fn.MAX(db.NetworkChange.status_id)
         )
     )
 

surykatka_status.py

@@ -19,8 +19,8 @@
 
 
 def reportStatus(db):
-    return db.Status.select().order_by(db.Status.id.desc()).limit(1)
+    return db.Status.select().order_by(db.Status.timestamp.desc()).limit(1)
 
 
 def logStatus(db, text):
-    return db.Status.create(text=text).id
+    return db.Status.create(text=text).timestamp

test_surykatka_status.py

@@ -30,17 +30,16 @@ class SurykatkaStatusTestCase(unittest.TestCase):
     def test_logStatus_insert(self):
         result = logStatus(self.db, "foo")
         assert self.db.Status.select().count() == 1
-        assert self.db.Status.get(self.db.Status.text == "foo").id == result
+        assert (
+            self.db.Status.get(self.db.Status.text == "foo").timestamp
+            == result
+        )
 
     def test_logStatus_insertTwice(self):
         result1 = logStatus(self.db, "foo")
         result2 = logStatus(self.db, "foo")
         assert self.db.Status.select().count() == 2
         assert result1 < result2
-        assert (
-            self.db.Status.get(self.db.Status.id == result1).timestamp
-            <= self.db.Status.get(self.db.Status.id == result2).timestamp
-        )
 
 
 def suite():