Commit 43d944f1 authored by Romain Courteaud's avatar Romain Courteaud

Allow to quickly check one site status with memory DB

parent b2fbb79f
...@@ -28,6 +28,10 @@ class WebBot: ...@@ -28,6 +28,10 @@ class WebBot:
def __init__(self, **kw): def __init__(self, **kw):
self.config = createConfiguration(**kw) self.config = createConfiguration(**kw)
def closeDB(self):
if hasattr(self, "_db"):
self._db.close()
def initDB(self): def initDB(self):
self._db = LogDB(self.config["SQLITE"]) self._db = LogDB(self.config["SQLITE"])
self._db.createTables() self._db.createTables()
...@@ -98,9 +102,6 @@ class WebBot: ...@@ -98,9 +102,6 @@ class WebBot:
# XXX Check HTTP Cache # XXX Check HTTP Cache
def status(self): def status(self):
# XXX
self.initDB()
# Report the bot status # Report the bot status
print("# STATUS") print("# STATUS")
status = reportStatus(self._db).get() status = reportStatus(self._db).get()
...@@ -150,7 +151,7 @@ class WebBot: ...@@ -150,7 +151,7 @@ class WebBot:
if server_ip not in server_ip_dict: if server_ip not in server_ip_dict:
server_ip_dict[server_ip] = [] server_ip_dict[server_ip] = []
if dns_change["domain"] not in server_ip_dict[server_ip]: if dns_change["domain"] not in server_ip_dict[server_ip]:
server_ip_dict[server_ip].append(dns_change["domain"]) server_ip_dict[server_ip].append(dns_change["domain"])
# Report the list of CDN status # Report the list of CDN status
query = reportNetwork( query = reportNetwork(
...@@ -206,11 +207,8 @@ class WebBot: ...@@ -206,11 +207,8 @@ class WebBot:
def stop(self): def stop(self):
self._running = False self._running = False
logStatus(self._db, "stop") logStatus(self._db, "stop")
if hasattr(self, "_db"):
self._db.close()
def run(self): def crawl(self):
self.initDB()
status_id = logStatus(self._db, "start") status_id = logStatus(self._db, "start")
logConfiguration(self._db, status_id, self.config) logConfiguration(self._db, status_id, self.config)
...@@ -228,9 +226,29 @@ class WebBot: ...@@ -228,9 +226,29 @@ class WebBot:
except: except:
# XXX Put traceback in the log? # XXX Put traceback in the log?
logStatus(self._db, "error") logStatus(self._db, "error")
self.stop()
raise raise
def run(self, mode):
if mode not in ["crawl", "status"]:
raise NotImplementedError("Unexpected mode: %s" % mode)
if self.config["SQLITE"] == ":memory:":
# Crawl/report are mandatory when using memory
mode = "all"
self.initDB()
try:
if mode in ["crawl", "all"]:
self.crawl()
if mode in ["status", "all"]:
self.status()
except:
self.closeDB()
raise
else:
self.closeDB()
def create_bot(**kw): def create_bot(**kw):
return WebBot(**kw) return WebBot(**kw)
...@@ -2,11 +2,16 @@ import click ...@@ -2,11 +2,16 @@ import click
import sys import sys
from urlchecker_bot import create_bot from urlchecker_bot import create_bot
@click.command(short_help="Runs url checker bot.") @click.command(short_help="Runs url checker bot.")
@click.option('--run', '-r', help="The bot operation to run.", @click.option(
show_default=True, "--run",
default='status', "-r",
type=click.Choice(['crawl', 'status'])) help="The bot operation mode to run.",
show_default=True,
default="status",
type=click.Choice(["crawl", "status"]),
)
@click.option( @click.option(
"--sqlite", "-s", help="The path of the sqlite DB. (default: :memory:)" "--sqlite", "-s", help="The path of the sqlite DB. (default: :memory:)"
) )
...@@ -32,12 +37,7 @@ def runUrlChecker(run, sqlite, dns, url, domain, configuration): ...@@ -32,12 +37,7 @@ def runUrlChecker(run, sqlite, dns, url, domain, configuration):
if dns: if dns:
mapping["DNS"] = dns mapping["DNS"] = dns
bot = create_bot(cfgfile=configuration, mapping=mapping) bot = create_bot(cfgfile=configuration, mapping=mapping)
if run == 'status': return bot.run(run)
return bot.status()
elif run == 'crawl':
return bot.run()
else:
raise NotImplementedError('Unexpected run: %s' % run)
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -29,6 +29,10 @@ def createConfiguration( ...@@ -29,6 +29,10 @@ def createConfiguration(
get_default_resolver().nameservers get_default_resolver().nameservers
) )
if config[CONFIG_SECTION]["SQLITE"] == ":memory:":
# Do not loop when using temporary DB
config[CONFIG_SECTION]["INTERVAL"] = "-1"
return config[CONFIG_SECTION] return config[CONFIG_SECTION]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment