whois: support of domain expiration

TODO:
* support more TLDs

setup.py

@@ -43,6 +43,7 @@ setup(
         "peewee>2.10.1",
         "click>=7.0",
         "dnspython",
+        "python-whois",
         "miniupnpc",
     ],
     extras_require={

src/surykatka/bot.py

@@ -28,6 +28,11 @@ from .dns import (
     reportDnsQuery,
     packDns,
 )
+from .domain import (
+    queryWhois,
+    reportWhoisQuery,
+    packDomain,
+)
 from .http import (
     getRootUrl,
     getUrlHostname,
@@ -73,6 +78,41 @@ def filterWarningStatus(status_dict, interval, not_critical_url_list):
     if not status_dict["bot_status"]:
         del status_dict["bot_status"]
 
+    for i in range(len(status_dict["whois"]) - 1, -1, -1):
+        expiration_date = status_dict["whois"][i]["expiration_date"]
+        if (expiration_date is None) or (
+            (expiration_date is not None)
+            and (
+                (60 * 60 * 24 * 14)
+                < (
+                    parsedate_to_datetime(expiration_date) - now
+                ).total_seconds()
+            )
+        ):
+            # Not handled whois entry hidden. Only check DNS warning in such case
+            # Warn 2 weeks before expiration
+            del status_dict["whois"][i]
+        else:
+            # Drop columns with too much info
+            del status_dict["whois"][i]["registrar"]
+            del status_dict["whois"][i]["whois_server"]
+            del status_dict["whois"][i]["creation_date"]
+            del status_dict["whois"][i]["updated_date"]
+            del status_dict["whois"][i]["name_servers"]
+            del status_dict["whois"][i]["whois_status"]
+            del status_dict["whois"][i]["emails"]
+            del status_dict["whois"][i]["dnssec"]
+            del status_dict["whois"][i]["name"]
+            del status_dict["whois"][i]["org"]
+            del status_dict["whois"][i]["address"]
+            del status_dict["whois"][i]["city"]
+            del status_dict["whois"][i]["state"]
+            del status_dict["whois"][i]["zipcode"]
+            del status_dict["whois"][i]["country"]
+
+    if not status_dict["whois"]:
+        del status_dict["whois"]
+
     for i in range(len(status_dict["dns_server"]) - 1, -1, -1):
         state = status_dict["dns_server"][i]["state"]
         if state == "open":
@@ -181,6 +221,24 @@ class WebBot:
             public_suffix_list=self.config["PUBLIC_SUFFIX"].split(),
         )
 
+    def calculateWhoisDomainList(self, domain_list):
+        # Calculate the top domain for whois
+        domain_list = domain_list.copy()
+        domain_list.sort(key=lambda x: x.count("."))
+        i = 0
+        while i < len(domain_list):
+            base_domain = ".%s" % domain_list[i]
+            j = i + 1
+            while j < len(domain_list):
+                sub_domain = domain_list[j]
+                if sub_domain.endswith(base_domain):
+                    domain_list.pop(j)
+                else:
+                    j += 1
+            i += 1
+
+        return domain_list
+
     def calculateNotCriticalUrlList(self):
         domain_list = self.config["DOMAIN"].split()
         url_list = self.config["URL"].split()
@@ -208,6 +266,13 @@ class WebBot:
         elapsed_moderate = float(self.config["ELAPSED_moderate"])
         # logPlatform(self._db, __version__, status_id)
 
+        # Get list of all domains
+        domain_list = self.calculateFullDomainList()
+        whois_domain_list = self.calculateWhoisDomainList(domain_list)
+
+        for whois_domain in whois_domain_list:
+            queryWhois(self._db, status_id, whois_domain)
+
         # Calculate the resolver list
         resolver_ip_list = getReachableResolverList(
             self._db, status_id, self.config["NAMESERVER"].split(), timeout
@@ -215,9 +280,6 @@ class WebBot:
         if not resolver_ip_list:
             return
 
-        # Get list of all domains
-        domain_list = self.calculateFullDomainList()
-
         # Get the list of server to check
         # XXX Check DNS expiration
         server_ip_dict = getDomainIpDict(
@@ -293,6 +355,42 @@ class WebBot:
                 {"text": bot_status.text, "date": rfc822(bot_status.timestamp)}
             )
 
+        domain_list = self.calculateFullDomainList()
+        whois_domain_list = self.calculateWhoisDomainList(domain_list)
+
+        # Report list of Whois query
+        query = reportWhoisQuery(self._db, domain=whois_domain_list)
+        result_dict["whois"] = []
+        for domain_change in query.dicts().iterator():
+            result_dict["whois"].append(
+                {
+                    "domain": domain_change["domain"],
+                    "date": rfc822(domain_change["status"]),
+                    "registrar": domain_change["registrar"],
+                    "whois_server": domain_change["whois_server"],
+                    "creation_date": rfc822(domain_change["creation_date"])
+                    if (domain_change["creation_date"] is not None)
+                    else None,
+                    "updated_date": rfc822(domain_change["updated_date"])
+                    if (domain_change["updated_date"] is not None)
+                    else None,
+                    "expiration_date": rfc822(domain_change["expiration_date"])
+                    if (domain_change["expiration_date"] is not None)
+                    else None,
+                    "name_servers": domain_change["name_servers"],
+                    "whois_status": domain_change["whois_status"],
+                    "emails": domain_change["emails"],
+                    "dnssec": domain_change["dnssec"],
+                    "name": domain_change["name"],
+                    "org": domain_change["org"],
+                    "address": domain_change["address"],
+                    "city": domain_change["city"],
+                    "state": domain_change["state"],
+                    "zipcode": domain_change["zipcode"],
+                    "country": domain_change["country"],
+                }
+            )
+
         # Report the list of DNS server status
         checked_resolver_ip_dict = {}
         query = reportNetwork(
@@ -315,7 +413,6 @@ class WebBot:
                 }
             )
 
-        domain_list = self.calculateFullDomainList()
         checked_domain_dict = {}
         # Report list of DNS query
         query = reportDnsQuery(
@@ -487,6 +584,7 @@ class WebBot:
     def pack(self):
         logStatus(self._db, "packing")
         packDns(self._db)
+        packDomain(self._db)
         packHttp(self._db)
         packNetwork(self._db)
         packSslCertificate(self._db)

src/surykatka/db.py

@@ -106,6 +106,30 @@ class LogDB:
                     (("resolver_ip", "domain", "rdtype", "status"), True),
                 )
 
+        class DomainChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            domain = peewee.TextField()
+            registrar = peewee.TextField(null=True)
+            whois_server = peewee.TextField(null=True)
+            creation_date = peewee.DateTimeField(null=True)
+            updated_date = peewee.DateTimeField(null=True)
+            expiration_date = peewee.DateTimeField(null=True)
+            name_servers = peewee.TextField(null=True)
+            whois_status = peewee.TextField(null=True)
+            emails = peewee.TextField(null=True)
+            dnssec = peewee.TextField(null=True)
+            name = peewee.TextField(null=True)
+            org = peewee.TextField(null=True)
+            address = peewee.TextField(null=True)
+            city = peewee.TextField(null=True)
+            state = peewee.TextField(null=True)
+            zipcode = peewee.TextField(null=True)
+            country = peewee.TextField(null=True)
+
+            class Meta:
+                primary_key = peewee.CompositeKey("status", "domain")
+                indexes = ((("domain", "status"), True),)
+
         class SslChange(BaseModel):
             status = peewee.ForeignKeyField(Status)
             ip = peewee.TextField()
@@ -140,13 +164,14 @@ class LogDB:
         self.PlatformChange = PlatformChange
         self.NetworkChange = NetworkChange
         self.DnsChange = DnsChange
+        self.DomainChange = DomainChange
         self.HttpCodeChange = HttpCodeChange
         self.SslChange = SslChange
 
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 5
+        expected_version = 6
         if db_version != expected_version:
             with self._db.transaction():
 
@@ -167,6 +192,9 @@ class LogDB:
                     # version 1 without SSL support
                     self._db.create_tables([self.SslChange])
 
+                if db_version <= 5:
+                    self._db.create_tables([self.DomainChange])
+
                 migrator = SqliteMigrator(self._db)
                 migration_list = []
 

src/surykatka/domain.py

+# Copyright (C) 2021  Nexedi SA and Contributors.
+#                     Romain Courteaud <romain@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import sys
+import os
+import whois
+from peewee import fn
+
+
+def reportWhoisQuery(db, domain=None):
+    query = (
+        db.DomainChange.select(db.DomainChange)
+        .group_by(db.DomainChange.domain)
+        .having(db.DomainChange.status_id == fn.MAX(db.DomainChange.status_id))
+    )
+
+    if domain is not None:
+        if type(domain) == list:
+            query = query.where(db.DomainChange.domain << domain)
+        else:
+            query = query.where(db.DomainChange.domain == domain)
+    return query
+
+
+def packDomain(db):
+    with db._db.atomic():
+        result = [x for x in reportWhoisQuery(db)]
+        for dns_change in result:
+            db.DomainChange.delete().where(
+                db.DomainChange.status_id != dns_change.status_id,
+                db.DomainChange.domain == dns_change.domain,
+            ).execute()
+
+
+def logWhoisQuery(
+    db,
+    status_id,
+    domain_text,
+    registrar,
+    whois_server,
+    creation_date,
+    updated_date,
+    expiration_date,
+    name_servers,
+    whois_status,
+    emails,
+    dnssec,
+    name,
+    org,
+    address,
+    city,
+    state,
+    zipcode,
+    country,
+):
+
+    with db._db.atomic():
+        try:
+            # Check previous parameter value
+            previous_entry = reportWhoisQuery(db, domain=domain_text).get()
+        except db.DomainChange.DoesNotExist:
+            previous_entry = None
+
+        if (
+            (previous_entry is None)
+            or (previous_entry.registrar != registrar)
+            or (previous_entry.whois_server != whois_server)
+            or (previous_entry.creation_date != creation_date)
+            or (previous_entry.updated_date != updated_date)
+            or (previous_entry.expiration_date != expiration_date)
+            or (previous_entry.name_servers != name_servers)
+            or (previous_entry.whois_status != whois_status)
+            or (previous_entry.emails != emails)
+            or (previous_entry.dnssec != dnssec)
+            or (previous_entry.name != name)
+            or (previous_entry.org != org)
+            or (previous_entry.address != address)
+            or (previous_entry.city != city)
+            or (previous_entry.state != state)
+            or (previous_entry.zipcode != zipcode)
+            or (previous_entry.country != country)
+        ):
+            previous_entry = db.DomainChange.create(
+                domain=domain_text,
+                registrar=registrar,
+                whois_server=whois_server,
+                creation_date=creation_date,
+                updated_date=updated_date,
+                expiration_date=expiration_date,
+                name_servers=name_servers,
+                whois_status=whois_status,
+                emails=emails,
+                dnssec=dnssec,
+                name=name,
+                org=org,
+                address=address,
+                city=city,
+                state=state,
+                zipcode=zipcode,
+                country=country,
+                status=status_id,
+            )
+
+    return previous_entry.status_id
+
+
+def queryWhois(db, status_id, domain_text):
+    # Hide lib message:
+    # Error trying to connect to socket: closing socket
+    _stdout = sys.stdout
+    sys.stdout = open(os.devnull, "w")
+    whois_dict = whois.whois(domain_text)
+    sys.stdout = _stdout
+
+    arg_list = []
+    for arg in [
+        whois_dict.registrar,
+        whois_dict.whois_server,
+        whois_dict.creation_date,
+        whois_dict.updated_date,
+        whois_dict.expiration_date,
+        whois_dict.name_servers,
+        whois_dict.status,
+        whois_dict.emails,
+        whois_dict.dnssec,
+        whois_dict.name,
+        whois_dict.org,
+        whois_dict.address,
+        whois_dict.city,
+        whois_dict.state,
+        whois_dict.zipcode,
+        whois_dict.country,
+    ]:
+        if type(arg) == list:
+            arg = arg[0]
+        arg_list.append(arg)
+
+    logWhoisQuery(db, status_id, domain_text, *arg_list)
+    return whois_dict

tests/test_db.py

@@ -92,10 +92,8 @@ class SurykatkaDBTestCase(unittest.TestCase):
     def setUp(self):
         self.db = LogDB(":memory:")
 
-    def test_createTable(self):
-        assert self.db._db.pragma("user_version") == 0
-        self.db.createTables()
-        assert self.db._db.pragma("user_version") == 5
+    def checkDatabaseStatus(self):
+        assert self.db._db.pragma("user_version") == 6
         assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
             self.db.HttpCodeChange
         )
@@ -105,9 +103,26 @@ class SurykatkaDBTestCase(unittest.TestCase):
         assert validate_schema(self.db.DnsChange).valid, validate_schema(
             self.db.DnsChange
         )
+        assert validate_schema(self.db.DomainChange).valid, validate_schema(
+            self.db.DomainChange
+        )
         assert validate_schema(self.db.NetworkChange).valid, validate_schema(
             self.db.NetworkChange
         )
+        assert validate_schema(self.db.PlatformChange).valid, validate_schema(
+            self.db.PlatformChange
+        )
+        assert validate_schema(
+            self.db.ConfigurationChange
+        ).valid, validate_schema(self.db.ConfigurationChange)
+        assert validate_schema(self.db.Status).valid, validate_schema(
+            self.db.Status
+        )
+
+    def test_createTable(self):
+        assert self.db._db.pragma("user_version") == 0
+        self.db.createTables()
+        self.checkDatabaseStatus()
 
     def test_downgrade(self):
         assert self.db._db.pragma("user_version") == 0
@@ -165,19 +180,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
             self.db._db.pragma("user_version", 1)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 5
-        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
-            self.db.HttpCodeChange
-        )
-        assert validate_schema(self.db.SslChange).valid, validate_schema(
-            self.db.SslChange
-        )
-        assert validate_schema(self.db.DnsChange).valid, validate_schema(
-            self.db.DnsChange
-        )
-        assert validate_schema(self.db.NetworkChange).valid, validate_schema(
-            self.db.NetworkChange
-        )
+        self.checkDatabaseStatus()
 
     def test_migrationFromVersion2(self):
         assert self.db._db.pragma("user_version") == 0
@@ -230,19 +233,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
             self.db._db.pragma("user_version", 2)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 5
-        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
-            self.db.HttpCodeChange
-        )
-        assert validate_schema(self.db.SslChange).valid, validate_schema(
-            self.db.SslChange
-        )
-        assert validate_schema(self.db.DnsChange).valid, validate_schema(
-            self.db.DnsChange
-        )
-        assert validate_schema(self.db.NetworkChange).valid, validate_schema(
-            self.db.NetworkChange
-        )
+        self.checkDatabaseStatus()
 
     def test_migrationFromVersion3(self):
         assert self.db._db.pragma("user_version") == 0
@@ -290,19 +281,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
             self.db._db.pragma("user_version", 3)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 5
-        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
-            self.db.HttpCodeChange
-        )
-        assert validate_schema(self.db.SslChange).valid, validate_schema(
-            self.db.SslChange
-        )
-        assert validate_schema(self.db.DnsChange).valid, validate_schema(
-            self.db.DnsChange
-        )
-        assert validate_schema(self.db.NetworkChange).valid, validate_schema(
-            self.db.NetworkChange
-        )
+        self.checkDatabaseStatus()
 
     def test_migrationFromVersion4(self):
         assert self.db._db.pragma("user_version") == 0
@@ -345,19 +324,28 @@ class SurykatkaDBTestCase(unittest.TestCase):
             self.db._db.pragma("user_version", 4)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 5
-        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
-            self.db.HttpCodeChange
-        )
-        assert validate_schema(self.db.SslChange).valid, validate_schema(
-            self.db.SslChange
-        )
-        assert validate_schema(self.db.DnsChange).valid, validate_schema(
-            self.db.DnsChange
-        )
-        assert validate_schema(self.db.NetworkChange).valid, validate_schema(
-            self.db.NetworkChange
-        )
+        self.checkDatabaseStatus()
+
+    def test_migrationFromVersion5(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Recreate version 5
+        with self.db._db.transaction():
+
+            self.db._db.create_tables(
+                [
+                    self.db.Status,
+                    self.db.ConfigurationChange,
+                    self.db.HttpCodeChange,
+                    self.db.NetworkChange,
+                    self.db.PlatformChange,
+                    self.db.DnsChange,
+                    self.db.SslChange,
+                ]
+            )
+            self.db._db.pragma("user_version", 5)
+
+        self.db.createTables()
+        self.checkDatabaseStatus()
 
 
 def suite():