further reorganization

respodns/__main__.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 
 from .db import RespoDB
+from .dns import getaddrs, detect_gfw
+from .ip_util import addr_to_int, read_ips
 from .ips import blocks, is_bogon
 from .structs import Options, Entry
-from .util import right_now, read_ips, getaddrs, detect_gfw, make_pooler
+from .util import right_now, make_pooler
 from asyncio import run, sleep
 from sys import argv, stdin, stderr, exit
 import respodns.checks as chk

respodns/db.py

@@ -1,6 +1,8 @@
 from .sql import create_table_statements, create_view_statements
 from .sql import table_triggers
-from .util import addr_to_int, AttrCheck
+from .tables import TException, TExecution, TAddress
+from .tables import TKind, TDomain, TRecord, TMessage
+from .ip_util import addr_to_int
 import storm.locals as rain
 
 class Execution:
@@ -18,65 +20,6 @@ class Execution:
         completed = exc_type is None
         self.db.finish_execution(self.execution, right_now(), completed)
 
-class TException(rain.Storm, AttrCheck):
-    __storm_table__ = "Exceptions"
-    exception_id = rain.Int("ExceptionId", primary=True)
-    name = rain.Unicode("Name")
-    fail = rain.Bool("Fail")
-
-class TExecution(rain.Storm, AttrCheck):
-    __storm_table__ = "Executions"
-    execution_id = rain.Int("ExecutionId", primary=True)
-    start_date = rain.DateTime("StartDate")
-    finish_date = rain.DateTime("FinishDate")
-    completed = rain.Bool("Completed")
-
-class TAddress(rain.Storm, AttrCheck):
-    __storm_table__ = "Ips"
-    address_id = rain.Int("IpId", primary=True)
-    str = rain.Unicode("AsStr")
-    ip = rain.Int("AsInt")
-    china = rain.Bool("China")
-    block_target = rain.Bool("BlockTarget")
-    server = rain.Bool("Server")
-    redirect_target = rain.Bool("RedirectTarget")
-    gfw_target = rain.Bool("GfwTarget")
-
-class TKind(rain.Storm, AttrCheck):
-    __storm_table__ = "Kinds"
-    kind_id = rain.Int("KindId", primary=True)
-    name = rain.Unicode("Name")
-    xxid = rain.Int("ExpectExceptionId")
-    exception = rain.Reference(xxid, "TException.exception_id")
-
-class TDomain(rain.Storm, AttrCheck):
-    __storm_table__ = "Domains"
-    domain_id = rain.Int("DomainId", primary=True)
-    name = rain.Unicode("Name")
-    kind_id = rain.Int("KindId")
-    kind = rain.Reference(kind_id, "TKind.kind_id")
-
-class TRecord(rain.Storm, AttrCheck):
-    __storm_table__ = "Records"
-    row_id = rain.Int("rowid", primary=True)
-    record_id = rain.Int("RecordId")
-    address_id = rain.Int("IpId")
-    address = rain.Reference(address_id, "TAddress.address_id")
-
-class TMessage(rain.Storm, AttrCheck):
-    __storm_table__ = "Messages"
-    message_id = rain.Int("MessageId", primary=True)
-    execution_id = rain.Int("ExecutionId")
-    server_id = rain.Int("ServerId")
-    domain_id = rain.Int("DomainId")
-    record_id = rain.Int("RecordId")
-    exception_id = rain.Int("ExceptionId")
-    execution = rain.Reference(execution_id, "TExecution.execution_id")
-    server = rain.Reference(server_id, "TAddress.address_id")
-    domain = rain.Reference(domain_id, "TDomain.domain_id")
-    #record = rain.Reference(record_id, "TRecord.record_id")
-    exception = rain.Reference(exception_id, "TException.exception_id")
-
 def apply_properties(obj, d):
     from storm.properties import PropertyColumn
     for k, v in d.items():

respodns/dns.py

@@ -0,0 +1,53 @@
+def detect_gfw(r, ip, check):
+    # attempt to detect interference from the Great Firewall of China.
+    #from .ips import china
+    #if r in china: return True
+
+    # class D or class E, neither of which are correct for a (public?) DNS.
+    #if int(r.partition(".")[0]) >= 224: return True
+
+    rs = lambda prefix: r.startswith(prefix)
+    de = lambda suffix: check.domain.endswith(suffix)
+    hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com")
+    if rs("31.13.") and not hosted: return True
+    if rs("66.220."): return True
+    if rs("69.63."): return True
+    if rs("69.171.") and not rs("69.171.250."): return True
+    if rs("74.86."): return True
+    if rs("75.126."): return True
+    if r == "64.13.192.74": return True
+    # more non-facebook GFW stuff:
+    # 31.13.64.33
+    # 31.13.70.1
+    # 31.13.70.20
+    # 31.13.76.16
+    # 31.13.86.1
+    # 173.252.110.21
+    # 192.99.140.48
+    # 199.16.156.40
+    # 199.16.158.190
+
+    return False
+
+async def getaddrs(server, domain, opts):
+    from .ip_util import ipkey
+    from dns.asyncresolver import Resolver
+    from dns.exception import Timeout
+    from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers
+
+    res = Resolver(configure=False)
+    if opts.impatient:
+        res.timeout = 5
+        res.lifetime = 2
+    res.nameservers = [server]
+    try:
+        ans = await res.resolve(domain, "A", search=False)
+    except NXDOMAIN:
+        return ["NXDOMAIN"]
+    except NoAnswer:
+        return ["NoAnswer"]
+    except NoNameservers:
+        return ["NoNameservers"]
+    except Timeout:
+        return ["Timeout"]
+    return sorted(set(rr.address for rr in ans.rrset), key=ipkey)

respodns/ip_util.py

@@ -0,0 +1,26 @@
+import re
+ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII)
+
+def read_ips(f):
+    # TODO: make async and more robust. (regex pls)
+    # TODO: does readlines() block if the pipe is left open i.e. user input?
+    for ip in f.readlines():
+        if "#" in ip:
+            ip, _, _ = ip.partition("#")
+        ip = ip.strip()
+        if ip.count(".") != 3:
+            continue
+        yield ip
+
+def addr_to_int(ip):
+    match = ipv4_pattern.fullmatch(ip)
+    assert match is not None, row
+    segs = list(map(int, match.group(1, 2, 3, 4)))
+    assert all(0 <= seg <= 255 for seg in segs), match.group(0)
+    numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3]
+    return numeric
+
+def ipkey(ip_string):
+    # this is more lenient than addr_to_int.
+    segs = [int(s) for s in ip_string.replace(":", ".").split(".")]
+    return sum(256**(3 - i) * seg for i, seg in enumerate(segs))

respodns/tables.py

@@ -0,0 +1,61 @@
+from .util import AttrCheck
+import storm.locals as rain
+
+class TException(rain.Storm, AttrCheck):
+    __storm_table__ = "Exceptions"
+    exception_id = rain.Int("ExceptionId", primary=True)
+    name = rain.Unicode("Name")
+    fail = rain.Bool("Fail")
+
+class TExecution(rain.Storm, AttrCheck):
+    __storm_table__ = "Executions"
+    execution_id = rain.Int("ExecutionId", primary=True)
+    start_date = rain.DateTime("StartDate")
+    finish_date = rain.DateTime("FinishDate")
+    completed = rain.Bool("Completed")
+
+class TAddress(rain.Storm, AttrCheck):
+    __storm_table__ = "Ips"
+    address_id = rain.Int("IpId", primary=True)
+    str = rain.Unicode("AsStr")
+    ip = rain.Int("AsInt")
+    china = rain.Bool("China")
+    block_target = rain.Bool("BlockTarget")
+    server = rain.Bool("Server")
+    redirect_target = rain.Bool("RedirectTarget")
+    gfw_target = rain.Bool("GfwTarget")
+
+class TKind(rain.Storm, AttrCheck):
+    __storm_table__ = "Kinds"
+    kind_id = rain.Int("KindId", primary=True)
+    name = rain.Unicode("Name")
+    xxid = rain.Int("ExpectExceptionId")
+    exception = rain.Reference(xxid, "TException.exception_id")
+
+class TDomain(rain.Storm, AttrCheck):
+    __storm_table__ = "Domains"
+    domain_id = rain.Int("DomainId", primary=True)
+    name = rain.Unicode("Name")
+    kind_id = rain.Int("KindId")
+    kind = rain.Reference(kind_id, "TKind.kind_id")
+
+class TRecord(rain.Storm, AttrCheck):
+    __storm_table__ = "Records"
+    row_id = rain.Int("rowid", primary=True)
+    record_id = rain.Int("RecordId")
+    address_id = rain.Int("IpId")
+    address = rain.Reference(address_id, "TAddress.address_id")
+
+class TMessage(rain.Storm, AttrCheck):
+    __storm_table__ = "Messages"
+    message_id = rain.Int("MessageId", primary=True)
+    execution_id = rain.Int("ExecutionId")
+    server_id = rain.Int("ServerId")
+    domain_id = rain.Int("DomainId")
+    record_id = rain.Int("RecordId")
+    exception_id = rain.Int("ExceptionId")
+    execution = rain.Reference(execution_id, "TExecution.execution_id")
+    server = rain.Reference(server_id, "TAddress.address_id")
+    domain = rain.Reference(domain_id, "TDomain.domain_id")
+    #record = rain.Reference(record_id, "TRecord.record_id")
+    exception = rain.Reference(exception_id, "TException.exception_id")

respodns/util.py

@@ -1,6 +1,3 @@
-import re
-ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII)
-
 rot13_mapping = {}
 for a, b, c, d in zip("anAN05", "mzMZ49", "naNA50", "zmZM94"):
     rot13_mapping.update(dict((chr(k), chr(v))
@@ -19,73 +16,6 @@ def nonsense_consistent(domain):
     length = rng.choices((9, 10, 11, 12), (4, 5, 3, 2))[0]
     return "".join(rng.choice(ascii_lowercase) for i in range(length))
 
-def detect_gfw(r, ip, check):
-    # attempt to detect interference from the Great Firewall of China.
-    #from .ips import china
-    #if r in china: return True
-
-    # class D or class E, neither of which are correct for a (public?) DNS.
-    #if int(r.partition(".")[0]) >= 224: return True
-
-    rs = lambda prefix: r.startswith(prefix)
-    de = lambda suffix: check.domain.endswith(suffix)
-    hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com")
-    if rs("31.13.") and not hosted: return True
-    if rs("66.220."): return True
-    if rs("69.63."): return True
-    if rs("69.171.") and not rs("69.171.250."): return True
-    if rs("74.86."): return True
-    if rs("75.126."): return True
-    if r == "64.13.192.74": return True
-    # more non-facebook GFW stuff:
-    # 31.13.64.33
-    # 31.13.70.1
-    # 31.13.70.20
-    # 31.13.76.16
-    # 31.13.86.1
-    # 173.252.110.21
-    # 192.99.140.48
-    # 199.16.156.40
-    # 199.16.158.190
-
-    return False
-
-async def getaddrs(server, domain, opts):
-    from dns.asyncresolver import Resolver
-    from dns.exception import Timeout
-    from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers
-    #from dns.resolver import Resolver
-
-    res = Resolver(configure=False)
-    if opts.impatient:
-        res.timeout = 5
-        res.lifetime = 2
-    res.nameservers = [server]
-    try:
-        #ans = res.resolve(domain, "A", search=False)
-        ans = await res.resolve(domain, "A", search=False)
-    except NXDOMAIN:
-        return ["NXDOMAIN"]
-    except NoAnswer:
-        return ["NoAnswer"]
-    except NoNameservers:
-        return ["NoNameservers"]
-    except Timeout:
-        return ["Timeout"]
-    #return list(set(rr.address for rr in ans.rrset))
-    return sorted(set(rr.address for rr in ans.rrset), key=ipkey)
-
-def read_ips(f):
-    # TODO: make async and more robust. (regex pls)
-    # TODO: does readlines() block if the pipe is left open i.e. user input?
-    for ip in f.readlines():
-        if "#" in ip:
-            ip, _, _ = ip.partition("#")
-        ip = ip.strip()
-        if ip.count(".") != 3:
-            continue
-        yield ip
-
 def rot13(s):
     return "".join(rot13_mapping.get(c, c) for c in s)
 
@@ -101,19 +31,6 @@ def head(n, it):
         pass
     return res
 
-def addr_to_int(ip):
-    match = ipv4_pattern.fullmatch(ip)
-    assert match is not None, row
-    segs = list(map(int, match.group(1, 2, 3, 4)))
-    assert all(0 <= seg <= 255 for seg in segs), match.group(0)
-    numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3]
-    return numeric
-
-def ipkey(ip_string):
-    # this is more lenient than addr_to_int.
-    segs = [int(s) for s in ip_string.replace(":", ".").split(".")]
-    return sum(256**(3 - i) * seg for i, seg in enumerate(segs))
-
 def taskize(item):
     from types import CoroutineType
     from asyncio import Task, create_task