Check DNS config

setup.py

@@ -4,7 +4,16 @@ setup(
     name="urlchecker",
     version="0.0.1",
     long_description=__doc__,
-    py_modules=["urlchecker-http", "urlchecker-cli"],
+    py_modules=[
+        "urlchecker_bot",
+        "urlchecker_cli",
+        "urlchecker_configuration",
+        "urlchecker_dns",
+        "urlchecker_http",
+        "urlchecker_network",
+        "urlchecker_platform",
+        "urlchecker_status",
+    ],
     include_package_data=False,
     zip_safe=True,
     install_requires=[
@@ -15,7 +24,6 @@ setup(
         "click>=7.0",
         "dnspython",
         "miniupnpc",
-        "msgpack-python",
     ],
     entry_points={
         "console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]

urlchecker_bot.py

+import time
+from urlchecker_db import LogDB
+from urlchecker_configuration import createConfiguration, logConfiguration
+from urlchecker_platform import logPlatform
+from urlchecker_status import logStatus
+from urlchecker_dns import getResolverDict, expandDomainList, getServerIpDict
+from urlchecker_http import getUrlHostname
+
+
+__version__ = "0.0.3"
+
+
+class BotError(Exception):
+    pass
+
+
+class WebBot:
+    def __init__(self, **kw):
+        self.config = createConfiguration(**kw)
+
+    def initDB(self, sqlite_path):
+        self._db = LogDB(sqlite_path)
+        self._db.createTables()
+
+    def iterateLoop(self):
+        status_id = logStatus(self._db, "loop")
+        logPlatform(self._db, __version__, status_id)
+
+        # Calculate the resolver list
+        resolver_dict = getResolverDict(
+            self._db, status_id, self.config["DNS"].split()
+        )
+        if not resolver_dict:
+            raise NotImplementedError("No resolver available")
+
+        # Calculate the full list of domain to check
+        domain_list = self.config["DOMAIN"].split()
+
+        # Extract the list of URL domains
+        url_list = self.config["URL"].split()
+        for url in url_list:
+            domain_list.append(getUrlHostname(url))
+        domain_list = list(set(domain_list))
+
+        # Expand with all parent domains
+        domain_list = expandDomainList(domain_list)
+        print(domain_list)
+
+        # Check if domain have an A record
+        server_id_dict = getServerIpDict(
+            self._db, status_id, resolver_dict, domain_list, "A"
+        )
+        print(server_id_dict)
+
+        # Check TCP port for the list of IP found
+        # If https ok, check SSL certificate
+        # Check HTTP Status
+
+    def stop(self):
+        self._running = False
+        logStatus(self._db, "stop")
+        if hasattr(self, "_db"):
+            self._db.close()
+
+    def run(self):
+        self.initDB(self.config["SQLITE"])
+        status_id = logStatus(self._db, "start")
+        logConfiguration(self._db, status_id, self.config)
+
+        self._running = True
+        try:
+            while self._running:
+                self.iterateLoop()
+                interval = int(self.config.get("INTERVAL"))
+                if interval < 0:
+                    self.stop()
+                else:
+                    time.sleep(interval)
+        except KeyboardInterrupt:
+            self.stop()
+        except:
+            # XXX Put traceback in the log?
+            self.stop()
+            logStatus(self._db, "error")
+            raise
+
+
+def create_bot(**kw):
+    return WebBot(**kw)

urlchecker_cli.py

 import click
 import sys
+from urlchecker_bot import create_bot
 
 
 @click.group()
@@ -9,15 +10,17 @@ def runUrlChecker():
 
 @runUrlChecker.command("bot", short_help="Runs url checker bot.")
 @click.option("--url", "-u", help="The url to check.")
+@click.option("--domain", "-m", help="The domain to check.")
 @click.option("--sqlite", "-s", help="The path of the sqlite DB.")
 @click.argument("configuration")
-def runWebBot(url, sqlite, configuration):
-    from urlchecker_http import create_bot
+def runWebBot(url, domain, sqlite, configuration):
 
-    click.echo("Running url checker bot")
+    # click.echo("Running url checker bot")
     mapping = {}
     if url:
         mapping["URL"] = url
+    if domain:
+        mapping["DOMAIN"] = url
     if sqlite:
         mapping["SQLITE"] = sqlite
     bot = create_bot(cfgfile=configuration, mapping=mapping)

urlchecker_configuration.py

+import configparser
+import os
+
+CONFIG_SECTION = "URLCHECKER"
+
+
+def createConfiguration(
+    envvar="URLCHECKER_SETTINGS", cfgfile=None, mapping=None
+):
+    config = configparser.ConfigParser(empty_lines_in_values=False)
+    # Default values
+    config[CONFIG_SECTION] = {
+        "INTERVAL": -1,
+        "DOMAIN": "",
+        "URL": "",
+        "DNS": "",
+    }
+
+    # User defined values
+    if (envvar is not None) and (envvar in os.environ):
+        config.read([os.environ.get(envvar)])
+    if cfgfile is not None:
+        config.read([cfgfile])
+    if mapping is not None:
+        config.read_dict({CONFIG_SECTION: mapping})
+
+    # Required values
+    for parameter in ["SQLITE"]:
+        if parameter not in config[CONFIG_SECTION]:
+            raise AttributeError("Config %s not defined" % parameter)
+
+    return config[CONFIG_SECTION]
+
+
+def logConfiguration(db, status_id, config):
+    with db._db.atomic():
+        for key, value in config.items():
+            try:
+                # Check previous parameter value
+                previous_value = (
+                    db.ConfigurationChange.select()
+                    .where(db.ConfigurationChange.parameter == key)
+                    .order_by(db.ConfigurationChange.status.desc())
+                    .get()
+                    .value
+                )
+            except db.ConfigurationChange.DoesNotExist:
+                previous_value = None
+
+            if previous_value != value:
+                db.ConfigurationChange.create(
+                    status=status_id, parameter=key, value=value
+                )

urlchecker_db.py

 import peewee
 from playhouse.migrate import migrate, SqliteMigrator
-import msgpack
 from playhouse.sqlite_ext import SqliteExtDatabase
 
 
-class MSGPackField(peewee.BlobField):
-    def db_value(self, value):
-        if value is not None:
-            return msgpack.dumps(value, use_bin_type=True)
-
-    def python_value(self, value):
-        if value is not None:
-            return msgpack.loads(value, encoding="utf-8")
-
-
 class LogDB:
     def __init__(self, sqlite_path):
         self._db = SqliteExtDatabase(
@@ -25,28 +14,70 @@ class LogDB:
             class Meta:
                 database = self._db
 
-        class Entry(BaseModel):
-            metadata = MSGPackField()
+        # This store the start, stop, loop time of the bot
+        # All other tables point to it to be able to group some info
+        class Status(BaseModel):
+            text = peewee.TextField(index=True)
+            timestamp = peewee.TimestampField(
+                index=True, constraints=[peewee.SQL("DEFAULT now")]
+            )
+
+        # Store the configuration modification
+        class ConfigurationChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            parameter = peewee.TextField(index=True)
+            value = peewee.TextField()
 
             class Meta:
-                order_by = ["id"]
+                primary_key = peewee.CompositeKey("status", "parameter")
 
-        class Query(BaseModel):
-            ip = peewee.TextField()
-            url = peewee.TextField()
-            status = peewee.IntegerField()
+        # Store the configuration modification
+        class PlatformChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            parameter = peewee.TextField(index=True)
+            value = peewee.TextField()
 
-        self.Entry = Entry
-        self.Query = Query
+            class Meta:
+                primary_key = peewee.CompositeKey("status", "parameter")
+
+        # Store remote network status
+        class NetworkChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            ip = peewee.TextField(index=True)
+            transport = peewee.TextField()
+            port = peewee.IntegerField()
+            state = peewee.TextField()
+            # class Meta:
+            #     primary_key = peewee.CompositeKey("status", "ip", "transport", "port")
+
+        class DnsChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            resolver_ip = peewee.TextField(index=True)
+            domain = peewee.TextField(index=True)
+            rdtype = peewee.TextField()
+            response = peewee.TextField()
+
+        self.Status = Status
+        self.ConfigurationChange = ConfigurationChange
+        self.PlatformChange = PlatformChange
+        self.NetworkChange = NetworkChange
+        self.DnsChange = DnsChange
 
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
         expected_version = 1
-        print(db_version)
         if db_version == 0:
             with self._db.transaction():
-                self._db.create_tables([self.Entry, self.Query])
+                self._db.create_tables(
+                    [
+                        self.Status,
+                        self.ConfigurationChange,
+                        self.NetworkChange,
+                        self.PlatformChange,
+                        self.DnsChange,
+                    ]
+                )
                 self._db.pragma("user_version", expected_version)
         elif db_version != expected_version:
             # migrator = SqliteMigrator(self._db)
@@ -66,23 +97,3 @@ class LogDB:
 
     def close(self):
         self._db.close()
-
-    def getEntry(self, id):
-        try:
-            return self.Entry.get(self.Entry.id == id)
-        except self.Entry.DoesNotExist:
-            return None
-
-    def storeEntry(self, **kw):
-        with self._db.atomic():
-            entry = self.Entry.create(metadata=kw)
-            id = entry.id
-
-        return id
-
-    def storeQuery(self, ip, url, status):
-        with self._db.atomic():
-            entry = self.Query.create(ip=ip, url=url, status=status)
-            id = entry.id
-
-        return id

urlchecker_dns.py

+from dns.resolver import get_default_resolver
+import dns.resolver
+import dns.name
+from urlchecker_network import logNetwork
+
+URL_TO_CHECK = "example.org"
+TIMEOUT = 2
+
+
+def logDnsQuery(db, status_id, resolver_ip, resolver, domain_text, rdtype):
+    # only A (and AAAA) has address property
+    assert rdtype == "A"
+
+    try:
+        answer_list = [
+            x.address
+            for x in resolver.query(
+                domain_text, rdtype, raise_on_no_answer=False
+            )
+        ]
+    except (
+        dns.resolver.NXDOMAIN,
+        dns.resolver.NoAnswer,
+        dns.exception.Timeout,
+        dns.resolver.NoNameservers,
+    ):
+        answer_list = []
+
+    answer_list.sort()
+    response = ", ".join(answer_list)
+
+    with db._db.atomic():
+        try:
+            # Check previous parameter value
+            previous_entry = (
+                db.DnsChange.select()
+                .where(
+                    db.DnsChange.resolver_ip == resolver_ip,
+                    db.DnsChange.domain == domain_text,
+                    db.DnsChange.rdtype == rdtype,
+                )
+                .order_by(db.DnsChange.status.desc())
+                .get()
+            )
+        except db.DnsChange.DoesNotExist:
+            previous_entry = None
+
+        if (previous_entry is None) or (previous_entry.response != response):
+            previous_entry = db.DnsChange.create(
+                resolver_ip=resolver_ip,
+                domain=domain_text,
+                rdtype=rdtype,
+                response=response,
+                status=status_id,
+            )
+
+    return answer_list
+
+
+def getResolverDict(db, status_id, resolver_ip_list):
+
+    # Create a list of resolver object
+    if len(resolver_ip_list) == 0:
+        resolver_ip_list = get_default_resolver().nameservers
+    resolver_dict = {}
+    for resolver_ip in resolver_ip_list:
+        resolver = dns.resolver.Resolver(configure=False)
+        resolver.nameservers.append(resolver_ip)
+        resolver.timeout = TIMEOUT
+        resolver.lifetime = TIMEOUT
+        resolver.edns = -1
+        resolver_dict[resolver_ip] = resolver
+
+    # Check the DNS server availability once
+    # to prevent using it later if it is down
+    resolver_tuple_list = [x for x in resolver_dict.items()]
+    for ip, resolver in resolver_tuple_list:
+        resolver_state = "open"
+        answer_list = logDnsQuery(
+            db, status_id, ip, resolver, URL_TO_CHECK, "A"
+        )
+
+        if len(answer_list) == 0:
+            # We expect a valid response
+            # Drop the DNS server...
+            resolver_dict.pop(ip)
+            resolver_state = "closed"
+        logNetwork(db, ip, "UDP", 53, resolver_state, status_id)
+
+    return resolver_dict
+
+
+def expandDomainList(domain_list):
+    for domain_text in domain_list:
+
+        dns_name = dns.name.from_text(domain_text)
+        if (len(dns_name.labels) - 1) > 2:
+            domain_list.append(dns_name.parent().to_text(omit_final_dot=True))
+
+    domain_list = list(set(domain_list))
+    domain_list.sort()
+    return domain_list
+
+
+def getServerIpDict(db, status_id, resolver_dict, domain_list, rdtype):
+    server_ip_dict = {}
+    for domain_text in domain_list:
+        for resolver_ip, resolver in resolver_dict.items():
+            answer_list = logDnsQuery(
+                db, status_id, resolver_ip, resolver, domain_text, rdtype
+            )
+            for address in answer_list:
+                if address not in server_ip_dict:
+                    server_ip_dict[address] = []
+                server_ip_dict[address].append(domain_text)
+    return server_ip_dict

urlchecker_network.py

+def logNetwork(db, ip, transport, port, state, status_id):
+
+    with db._db.atomic():
+        try:
+            # Check previous parameter value
+            previous_entry = (
+                db.NetworkChange.select()
+                .where(
+                    db.NetworkChange.ip == ip,
+                    db.NetworkChange.transport == transport,
+                    db.NetworkChange.port == port,
+                )
+                .order_by(db.NetworkChange.status.desc())
+                .get()
+            )
+        except db.NetworkChange.DoesNotExist:
+            previous_entry = None
+
+        if (previous_entry is None) or (previous_entry.state != state):
+            previous_entry = db.NetworkChange.create(
+                status=status_id,
+                ip=ip,
+                transport=transport,
+                port=port,
+                state=state,
+            )
+        return previous_entry.id

urlchecker_platform.py

+import miniupnpc
+from dns.resolver import get_default_resolver
+import platform
+import socket
+
+
+def checkPlatform(version):
+    config = {
+        "platform": platform.platform(),
+        "python_build": platform.python_build(),
+        "python_compiler": platform.python_compiler(),
+        "python_branch": platform.python_branch(),
+        "python_implementation": platform.python_implementation(),
+        "python_revision": platform.python_revision(),
+        "python_version": platform.python_version(),
+        "hostname": socket.gethostname(),
+        "version": version,
+    }
+
+    config["resolvers"] = get_default_resolver().nameservers
+
+    u = miniupnpc.UPnP()
+    u.discoverdelay = 1000
+    u.discover()
+    try:
+        u.selectigd()
+        config["ip"] = u.externalipaddress()
+    except Exception:
+        config["ip"] = None
+    return config
+
+
+def logPlatform(db, version, status_id):
+    config = checkPlatform(version)
+    with db._db.atomic():
+        for key, value in config.items():
+            value = str(value)
+            try:
+                # Check previous parameter value
+                previous_value = (
+                    db.PlatformChange.select()
+                    .where(db.PlatformChange.parameter == key)
+                    .order_by(db.PlatformChange.status.desc())
+                    .get()
+                    .value
+                )
+            except db.PlatformChange.DoesNotExist:
+                previous_value = None
+
+            if previous_value != value:
+                db.PlatformChange.create(
+                    status=status_id, parameter=key, value=value
+                )

urlchecker_status.py

+def logStatus(db, text):
+    return db.Status.create(text=text).id