Commit 1c4c8adf authored by Romain Courteaud's avatar Romain Courteaud

Proto sqlite handler

parent b4e222c1
......@@ -15,6 +15,7 @@ setup(
"click>=7.0",
"dnspython",
"miniupnpc",
"msgpack-python",
],
entry_points={
"console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]
......
......@@ -9,12 +9,13 @@ def runUrlChecker():
@runUrlChecker.command("bot", short_help="Runs url checker bot.")
@click.argument("url")
def runWebBot(url):
@click.argument("sqlite_path")
def runWebBot(url, sqlite_path):
from urlchecker_http import WebBot
click.echo("Running url checker bot")
bot = WebBot()
return bot.run(url)
return bot.run(url, sqlite_path)
if __name__ == "__main__":
......
import peewee
from playhouse.migrate import migrate, SqliteMigrator
import msgpack
from playhouse.sqlite_ext import SqliteExtDatabase
class MSGPackField(peewee.BlobField):
def db_value(self, value):
if value is not None:
return msgpack.dumps(value, use_bin_type=True)
def python_value(self, value):
if value is not None:
return msgpack.loads(value, encoding="utf-8")
class LogDB:
def __init__(self, sqlite_path):
self._db = SqliteExtDatabase(
sqlite_path, pragmas=(("journal_mode", "WAL"),)
)
self._db.connect()
class BaseModel(peewee.Model):
class Meta:
database = self._db
class Entry(BaseModel):
metadata = MSGPackField()
class Meta:
order_by = ["id"]
self.Entry = Entry
def createTables(self):
# http://www.sqlite.org/pragma.html#pragma_user_version
db_version = self._db.pragma("user_version")
expected_version = 1
print(db_version)
if db_version == 0:
with self._db.transaction():
self._db.create_tables([self.Entry])
self._db.pragma("user_version", expected_version)
elif db_version != expected_version:
migrator = SqliteMigrator(self._db)
migration_list = []
sql_query_list = []
if migration_list or sql_query_list:
with self._db.transaction():
if migration_list:
migrate(*migration_list)
if sql_query_list:
for sql_query in sql_query_list:
self._db.execute_sql(
sql_query, require_commit=False
)
self._db.pragma("user_version", expected_version)
def close(self):
self._db.close()
def getEntry(self, id):
try:
return self.Entry.get(self.Entry.id == id)
except self.Entry.DoesNotExist:
return None
def storeEntry(self, **kw):
with self._db.atomic():
entry = self.Entry.create(metadata=kw)
id = entry.id
return id
......@@ -8,6 +8,7 @@ from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
import dns.resolver
import miniupnpc
import platform
from urlchecker_db import LogDB
__version__ = "0.0.3"
......@@ -20,6 +21,10 @@ class BotError(Exception):
class WebBot:
def initDB(self, sqlite_path):
self._db = LogDB(sqlite_path)
self._db.createTables()
def getUserAgent(self):
return "%s/%s (+%s)" % (
"URLCHECKER",
......@@ -105,8 +110,11 @@ class WebBot:
print(ip, hostname, response.status_code)
def run(self, url):
def run(self, url, sqlite_path):
print(time.strftime("%Y-%m-%d %H:%M:%S"))
self.initDB(sqlite_path)
self._db.storeEntry(platform=platform.platform())
print("Platform", platform.platform())
print(
"Python",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment