Proto sqlite handler

setup.py

@@ -15,6 +15,7 @@ setup(
         "click>=7.0",
         "dnspython",
         "miniupnpc",
+        "msgpack-python",
     ],
     entry_points={
         "console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]

urlchecker_cli.py

@@ -9,12 +9,13 @@ def runUrlChecker():
 
 @runUrlChecker.command("bot", short_help="Runs url checker bot.")
 @click.argument("url")
-def runWebBot(url):
+@click.argument("sqlite_path")
+def runWebBot(url, sqlite_path):
     from urlchecker_http import WebBot
 
     click.echo("Running url checker bot")
     bot = WebBot()
-    return bot.run(url)
+    return bot.run(url, sqlite_path)
 
 
 if __name__ == "__main__":

urlchecker_db.py

+import peewee
+from playhouse.migrate import migrate, SqliteMigrator
+import msgpack
+from playhouse.sqlite_ext import SqliteExtDatabase
+
+
+class MSGPackField(peewee.BlobField):
+    def db_value(self, value):
+        if value is not None:
+            return msgpack.dumps(value, use_bin_type=True)
+
+    def python_value(self, value):
+        if value is not None:
+            return msgpack.loads(value, encoding="utf-8")
+
+
+class LogDB:
+    def __init__(self, sqlite_path):
+        self._db = SqliteExtDatabase(
+            sqlite_path, pragmas=(("journal_mode", "WAL"),)
+        )
+        self._db.connect()
+
+        class BaseModel(peewee.Model):
+            class Meta:
+                database = self._db
+
+        class Entry(BaseModel):
+            metadata = MSGPackField()
+
+            class Meta:
+                order_by = ["id"]
+
+        self.Entry = Entry
+
+    def createTables(self):
+        # http://www.sqlite.org/pragma.html#pragma_user_version
+        db_version = self._db.pragma("user_version")
+        expected_version = 1
+        print(db_version)
+        if db_version == 0:
+            with self._db.transaction():
+                self._db.create_tables([self.Entry])
+                self._db.pragma("user_version", expected_version)
+        elif db_version != expected_version:
+            migrator = SqliteMigrator(self._db)
+            migration_list = []
+            sql_query_list = []
+
+            if migration_list or sql_query_list:
+                with self._db.transaction():
+                    if migration_list:
+                        migrate(*migration_list)
+                    if sql_query_list:
+                        for sql_query in sql_query_list:
+                            self._db.execute_sql(
+                                sql_query, require_commit=False
+                            )
+                    self._db.pragma("user_version", expected_version)
+
+    def close(self):
+        self._db.close()
+
+    def getEntry(self, id):
+        try:
+            return self.Entry.get(self.Entry.id == id)
+        except self.Entry.DoesNotExist:
+            return None
+
+    def storeEntry(self, **kw):
+        with self._db.atomic():
+            entry = self.Entry.create(metadata=kw)
+            id = entry.id
+
+        return id

urlchecker_http.py

@@ -8,6 +8,7 @@ from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
 import dns.resolver
 import miniupnpc
 import platform
+from urlchecker_db import LogDB
 
 
 __version__ = "0.0.3"
@@ -20,6 +21,10 @@ class BotError(Exception):
 
 
 class WebBot:
+    def initDB(self, sqlite_path):
+        self._db = LogDB(sqlite_path)
+        self._db.createTables()
+
     def getUserAgent(self):
         return "%s/%s (+%s)" % (
             "URLCHECKER",
@@ -105,8 +110,11 @@ class WebBot:
 
             print(ip, hostname, response.status_code)
 
-    def run(self, url):
+    def run(self, url, sqlite_path):
         print(time.strftime("%Y-%m-%d %H:%M:%S"))
+        self.initDB(sqlite_path)
+
+        self._db.storeEntry(platform=platform.platform())
         print("Platform", platform.platform())
         print(
             "Python",
@@ -126,9 +134,9 @@ class WebBot:
         u.discover()
         u.selectigd()
         try:
-          print("external ip: {}".format(u.externalipaddress()))
+            print("external ip: {}".format(u.externalipaddress()))
         except Exception:
-          pass
+            pass
 
         try:
             self.check(url)