Organize code in subdirectory

CONTRIBUTE.md

@@ -6,14 +6,15 @@
 python3 -m venv venv
 . venv/bin/activate
 pip install --upgrade --editable ".[dev]"
+py.test
 urlchecker -u URL
 ```
 
 ## Check the code
 
 ```
-pyflakes *py
-black -t py37 -l 79 *py
+pyflakes src/surykatka/*py tests/*py
+black -t py37 -l 79 src/surykatka/*py tests/*py
 ```
 
 ## SNI Support

setup.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
     name="surykatka",
@@ -27,17 +27,8 @@ setup(
     author="Nexedi",
     author_email="romain@nexedi.com",
     long_description=__doc__,
-    py_modules=[
-        "surykatka_bot",
-        "surykatka_cli",
-        "surykatka_configuration",
-        "surykatka_dns",
-        "surykatka_http",
-        "surykatka_network",
-        "surykatka_platform",
-        "surykatka_status",
-        "surykatka_db",
-    ],
+    packages=find_packages("src"),
+    package_dir={"": "src"},
     include_package_data=False,
     zip_safe=True,
     install_requires=[
@@ -53,6 +44,6 @@ setup(
         "dev": ["pytest", "black", "pyflakes", "mock", "httpretty"]
     },
     entry_points={
-        "console_scripts": ["surykatka=surykatka_cli:runSurykatka "]
+        "console_scripts": ["surykatka=surykatka.cli:runSurykatka "]
     },
 )

surykatka_bot.py → src/surykatka/bot.py

@@ -18,22 +18,17 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import time
-from surykatka_db import LogDB
-from surykatka_configuration import createConfiguration, logConfiguration
-from surykatka_status import logStatus, reportStatus
-from surykatka_dns import (
+from .db import LogDB
+from .configuration import createConfiguration, logConfiguration
+from .status import logStatus, reportStatus
+from .dns import (
     getReachableResolverList,
     expandDomainList,
     getDomainIpDict,
     reportDnsQuery,
 )
-from surykatka_http import (
-    getRootUrl,
-    getUrlHostname,
-    checkHttpStatus,
-    reportHttp,
-)
-from surykatka_network import isTcpPortOpen, reportNetwork
+from .http import getRootUrl, getUrlHostname, checkHttpStatus, reportHttp
+from .network import isTcpPortOpen, reportNetwork
 import json
 import email.utils
 from collections import OrderedDict

surykatka_cli.py → src/surykatka/cli.py

@@ -19,7 +19,7 @@
 
 import click
 import sys
-from surykatka_bot import create_bot
+from .bot import create_bot
 
 
 @click.command(short_help="Runs surykatka bot.")

surykatka_dns.py → src/surykatka/dns.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import dns
-from surykatka_network import logNetwork
+from .network import logNetwork
 from peewee import fn
 
 URL_TO_CHECK = "example.org"

test_surykatka_bot.py → tests/test_bot.py

@@ -18,10 +18,10 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_bot import WebBot
+from surykatka.bot import WebBot
 import mock
-from test_surykatka_dns import MockAnswer
-import surykatka_dns
+from test_dns import MockAnswer
+import surykatka.dns
 
 
 def checkNetworkChange(bot, result_list):
@@ -58,13 +58,13 @@ class SurykatkaBotTestCase(unittest.TestCase):
     def test_emptyConfiguration(self):
         resolver_ip = "192.168.0.254"
 
-        resolver = surykatka_dns.dns.resolver.Resolver(configure=False)
+        resolver = surykatka.dns.dns.resolver.Resolver(configure=False)
         resolver.nameservers.append(resolver_ip)
 
         with mock.patch(
-            "surykatka_configuration.get_default_resolver"
+            "surykatka.configuration.get_default_resolver"
         ) as mock_get_default_resolver, mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_get_default_resolver.return_value = resolver
 
@@ -87,17 +87,17 @@ class SurykatkaBotTestCase(unittest.TestCase):
     def test_oneNameserverOneDomainOneIp(self):
         resolver_ip = "127.0.0.1"
 
-        resolver = surykatka_dns.dns.resolver.Resolver(configure=False)
+        resolver = surykatka.dns.dns.resolver.Resolver(configure=False)
         resolver.nameservers.append(resolver_ip)
 
         with mock.patch(
-            "surykatka_configuration.get_default_resolver"
+            "surykatka.configuration.get_default_resolver"
         ) as mock_get_default_resolver, mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_get_default_resolver.return_value = resolver
@@ -142,11 +142,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.return_value = [MockAnswer("1.2.3.4")]
@@ -194,11 +194,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.return_value = [MockAnswer("1.2.3.4")]
@@ -239,11 +239,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.return_value = [
@@ -295,11 +295,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.return_value = [MockAnswer("1.2.3.4")]
@@ -340,11 +340,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.return_value = [MockAnswer("1.2.3.4")]
@@ -383,11 +383,11 @@ class SurykatkaBotTestCase(unittest.TestCase):
         bot.initDB()
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query, mock.patch(
-            "surykatka_network.socket.socket"
+            "surykatka.network.socket.socket"
         ) as mock_socket, mock.patch(
-            "surykatka_http.request"
+            "surykatka.http.request"
         ) as mock_request:
 
             mock_query.side_effect = [[MockAnswer("1.2.3.4")], []]

test_surykatka_db.py → tests/test_db.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_db import LogDB
+from surykatka.db import LogDB
 
 
 class SurykatkaDBTestCase(unittest.TestCase):

test_surykatka_dns.py → tests/test_dns.py

@@ -18,10 +18,10 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_db import LogDB
+from surykatka.db import LogDB
 import peewee
-import surykatka_dns
-from surykatka_dns import (
+import surykatka.dns
+from surykatka.dns import (
     expandDomainList,
     logDnsQuery,
     buildResolver,
@@ -29,7 +29,7 @@ from surykatka_dns import (
     getReachableResolverList,
     getDomainIpDict,
 )
-from surykatka_status import logStatus
+from surykatka.status import logStatus
 import mock
 
 
@@ -245,7 +245,7 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_query.return_value = [
                 MockAnswer("4.3.2.1"),
@@ -286,11 +286,11 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
 
             def sideEffect(*args, **kw):
-                raise surykatka_dns.dns.resolver.NXDOMAIN()
+                raise surykatka.dns.dns.resolver.NXDOMAIN()
 
             mock_query.side_effect = sideEffect
             result = queryDNS(self.db, status_id, resolver_ip, domain, rdtype)
@@ -315,11 +315,11 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
 
             def sideEffect(*args, **kw):
-                raise surykatka_dns.dns.resolver.NoAnswer()
+                raise surykatka.dns.dns.resolver.NoAnswer()
 
             mock_query.side_effect = sideEffect
             result = queryDNS(self.db, status_id, resolver_ip, domain, rdtype)
@@ -344,11 +344,11 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
 
             def sideEffect(*args, **kw):
-                raise surykatka_dns.dns.exception.Timeout()
+                raise surykatka.dns.dns.exception.Timeout()
 
             mock_query.side_effect = sideEffect
             result = queryDNS(self.db, status_id, resolver_ip, domain, rdtype)
@@ -373,11 +373,11 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
 
             def sideEffect(*args, **kw):
-                raise surykatka_dns.dns.resolver.NoNameservers()
+                raise surykatka.dns.dns.resolver.NoNameservers()
 
             mock_query.side_effect = sideEffect
             result = queryDNS(self.db, status_id, resolver_ip, domain, rdtype)
@@ -405,7 +405,7 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_query.return_value = [
                 MockAnswer("4.3.2.1"),
@@ -442,11 +442,11 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
 
             def sideEffect(*args, **kw):
-                raise surykatka_dns.dns.exception.Timeout()
+                raise surykatka.dns.dns.exception.Timeout()
 
             mock_query.side_effect = sideEffect
 
@@ -484,7 +484,7 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_query.return_value = [
                 MockAnswer("4.3.2.1"),
@@ -513,7 +513,7 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_query.side_effect = [
                 [MockAnswer("4.3.2.1"), MockAnswer("1.2.3.4")],
@@ -546,7 +546,7 @@ class SurykatkaDNSTestCase(unittest.TestCase):
         status_id = logStatus(self.db, "foo")
 
         with mock.patch(
-            "surykatka_dns.dns.resolver.Resolver.query"
+            "surykatka.dns.dns.resolver.Resolver.query"
         ) as mock_query:
             mock_query.side_effect = [
                 [MockAnswer("4.3.2.1"), MockAnswer("1.2.3.4")],

test_surykatka_http.py → tests/test_http.py

@@ -18,9 +18,9 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_db import LogDB
-import surykatka_http
-from surykatka_http import (
+from surykatka.db import LogDB
+import surykatka.http
+from surykatka.http import (
     getUrlHostname,
     getRootUrl,
     getUserAgent,
@@ -28,7 +28,7 @@ from surykatka_http import (
     logHttpStatus,
     checkHttpStatus,
 )
-from surykatka_status import logStatus
+from surykatka.status import logStatus
 import httpretty
 import mock
 import peewee
@@ -68,7 +68,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
     ################################################
     def test_request_arguments(self):
         url_to_proxy = "http://example.org/"
-        with mock.patch("surykatka_http.requests.request") as mock_request:
+        with mock.patch("surykatka.http.requests.request") as mock_request:
             request(url_to_proxy)
             assert mock_request.call_count == 1
             mock_request.assert_called_with(
@@ -128,10 +128,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
     def test_request_connectionError(self):
         url_to_proxy = "http://example.org/"
         httpretty.register_uri(httpretty.GET, url_to_proxy)
-        with mock.patch("surykatka_http.requests.request") as mock_request:
+        with mock.patch("surykatka.http.requests.request") as mock_request:
 
             def sideEffect(*args, **kw):
-                raise surykatka_http.requests.exceptions.ConnectionError()
+                raise surykatka.http.requests.exceptions.ConnectionError()
 
             mock_request.side_effect = sideEffect
             response = request(url_to_proxy)
@@ -141,10 +141,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
     def test_request_timeout(self):
         url_to_proxy = "http://example.org/"
         httpretty.register_uri(httpretty.GET, url_to_proxy)
-        with mock.patch("surykatka_http.requests.request") as mock_request:
+        with mock.patch("surykatka.http.requests.request") as mock_request:
 
             def sideEffect(*args, **kw):
-                raise surykatka_http.requests.exceptions.Timeout()
+                raise surykatka.http.requests.exceptions.Timeout()
 
             mock_request.side_effect = sideEffect
             response = request(url_to_proxy)
@@ -154,10 +154,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
     def test_request_tooManyRedirect(self):
         url_to_proxy = "http://example.org/"
         httpretty.register_uri(httpretty.GET, url_to_proxy)
-        with mock.patch("surykatka_http.requests.request") as mock_request:
+        with mock.patch("surykatka.http.requests.request") as mock_request:
 
             def sideEffect(*args, **kw):
-                raise surykatka_http.requests.exceptions.TooManyRedirects()
+                raise surykatka.http.requests.exceptions.TooManyRedirects()
 
             mock_request.side_effect = sideEffect
             response = request(url_to_proxy)
@@ -167,10 +167,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
     def test_request_sslError(self):
         url_to_proxy = "http://example.org/"
         httpretty.register_uri(httpretty.GET, url_to_proxy)
-        with mock.patch("surykatka_http.requests.request") as mock_request:
+        with mock.patch("surykatka.http.requests.request") as mock_request:
 
             def sideEffect(*args, **kw):
-                raise surykatka_http.requests.exceptions.SSLError()
+                raise surykatka.http.requests.exceptions.SSLError()
 
             mock_request.side_effect = sideEffect
             response = request(url_to_proxy)
@@ -322,7 +322,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         bot_version = 2
 
         status_id = logStatus(self.db, "foo")
-        with mock.patch("surykatka_http.request") as mock_request:
+        with mock.patch("surykatka.http.request") as mock_request:
             checkHttpStatus(self.db, status_id, url, ip, bot_version)
 
             assert mock_request.call_count == 1

test_surykatka_network.py → tests/test_network.py

@@ -18,10 +18,10 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_db import LogDB
-import surykatka_network
-from surykatka_network import logNetwork, isTcpPortOpen
-from surykatka_status import logStatus
+from surykatka.db import LogDB
+import surykatka.network
+from surykatka.network import logNetwork, isTcpPortOpen
+from surykatka.status import logStatus
 import mock
 import peewee
 
@@ -189,7 +189,7 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
             result = isTcpPortOpen(self.db, ip, port, status_id)
             assert mock_socket.call_count == 1
 
@@ -214,7 +214,7 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
                 raise ConnectionRefusedError()
@@ -245,10 +245,10 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
-                raise surykatka_network.socket.timeout()
+                raise surykatka.network.socket.timeout()
 
             mock_socket.return_value.connect.side_effect = sideEffect
 
@@ -276,10 +276,10 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
-                raise OSError(surykatka_network.errno.EHOSTUNREACH, "foo")
+                raise OSError(surykatka.network.errno.EHOSTUNREACH, "foo")
 
             mock_socket.return_value.connect.side_effect = sideEffect
 
@@ -307,10 +307,10 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
-                raise OSError(surykatka_network.errno.ENETUNREACH, "foo")
+                raise OSError(surykatka.network.errno.ENETUNREACH, "foo")
 
             mock_socket.return_value.connect.side_effect = sideEffect
 
@@ -338,7 +338,7 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
                 raise OSError()
@@ -367,7 +367,7 @@ class SurykatkaNetworkTestCase(unittest.TestCase):
         port = 1234
         status_id = logStatus(self.db, "foo")
 
-        with mock.patch("surykatka_network.socket.socket") as mock_socket:
+        with mock.patch("surykatka.network.socket.socket") as mock_socket:
 
             def sideEffect(*args, **kw):
                 raise Exception()

test_surykatka_status.py → tests/test_status.py

@@ -18,8 +18,8 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import unittest
-from surykatka_db import LogDB
-from surykatka_status import logStatus
+from surykatka.db import LogDB
+from surykatka.status import logStatus
 
 
 class SurykatkaStatusTestCase(unittest.TestCase):