From 85045f397f798ea379a0e8db763273f7c70225a8 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sat, 29 Aug 2020 15:04:56 +0200 Subject: [PATCH] further reorganization --- respodns/__main__.py | 4 ++- respodns/db.py | 63 ++------------------------------- respodns/dns.py | 53 ++++++++++++++++++++++++++++ respodns/ip_util.py | 26 ++++++++++++++ respodns/tables.py | 61 ++++++++++++++++++++++++++++++++ respodns/util.py | 83 -------------------------------------------- 6 files changed, 146 insertions(+), 144 deletions(-) create mode 100644 respodns/dns.py create mode 100644 respodns/ip_util.py create mode 100644 respodns/tables.py diff --git a/respodns/__main__.py b/respodns/__main__.py index 67b253c..5d83e5a 100644 --- a/respodns/__main__.py +++ b/respodns/__main__.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 from .db import RespoDB +from .dns import getaddrs, detect_gfw +from .ip_util import addr_to_int, read_ips from .ips import blocks, is_bogon from .structs import Options, Entry -from .util import right_now, read_ips, getaddrs, detect_gfw, make_pooler +from .util import right_now, make_pooler from asyncio import run, sleep from sys import argv, stdin, stderr, exit import respodns.checks as chk diff --git a/respodns/db.py b/respodns/db.py index 550db9c..ebd3631 100644 --- a/respodns/db.py +++ b/respodns/db.py @@ -1,6 +1,8 @@ from .sql import create_table_statements, create_view_statements from .sql import table_triggers -from .util import addr_to_int, AttrCheck +from .tables import TException, TExecution, TAddress +from .tables import TKind, TDomain, TRecord, TMessage +from .ip_util import addr_to_int import storm.locals as rain class Execution: @@ -18,65 +20,6 @@ class Execution: completed = exc_type is None self.db.finish_execution(self.execution, right_now(), completed) -class TException(rain.Storm, AttrCheck): - __storm_table__ = "Exceptions" - exception_id = rain.Int("ExceptionId", primary=True) - name = rain.Unicode("Name") - fail = rain.Bool("Fail") - -class TExecution(rain.Storm, AttrCheck): - __storm_table__ = "Executions" - execution_id = rain.Int("ExecutionId", primary=True) - start_date = rain.DateTime("StartDate") - finish_date = rain.DateTime("FinishDate") - completed = rain.Bool("Completed") - -class TAddress(rain.Storm, AttrCheck): - __storm_table__ = "Ips" - address_id = rain.Int("IpId", primary=True) - str = rain.Unicode("AsStr") - ip = rain.Int("AsInt") - china = rain.Bool("China") - block_target = rain.Bool("BlockTarget") - server = rain.Bool("Server") - redirect_target = rain.Bool("RedirectTarget") - gfw_target = rain.Bool("GfwTarget") - -class TKind(rain.Storm, AttrCheck): - __storm_table__ = "Kinds" - kind_id = rain.Int("KindId", primary=True) - name = rain.Unicode("Name") - xxid = rain.Int("ExpectExceptionId") - exception = rain.Reference(xxid, "TException.exception_id") - -class TDomain(rain.Storm, AttrCheck): - __storm_table__ = "Domains" - domain_id = rain.Int("DomainId", primary=True) - name = rain.Unicode("Name") - kind_id = rain.Int("KindId") - kind = rain.Reference(kind_id, "TKind.kind_id") - -class TRecord(rain.Storm, AttrCheck): - __storm_table__ = "Records" - row_id = rain.Int("rowid", primary=True) - record_id = rain.Int("RecordId") - address_id = rain.Int("IpId") - address = rain.Reference(address_id, "TAddress.address_id") - -class TMessage(rain.Storm, AttrCheck): - __storm_table__ = "Messages" - message_id = rain.Int("MessageId", primary=True) - execution_id = rain.Int("ExecutionId") - server_id = rain.Int("ServerId") - domain_id = rain.Int("DomainId") - record_id = rain.Int("RecordId") - exception_id = rain.Int("ExceptionId") - execution = rain.Reference(execution_id, "TExecution.execution_id") - server = rain.Reference(server_id, "TAddress.address_id") - domain = rain.Reference(domain_id, "TDomain.domain_id") - #record = rain.Reference(record_id, "TRecord.record_id") - exception = rain.Reference(exception_id, "TException.exception_id") - def apply_properties(obj, d): from storm.properties import PropertyColumn for k, v in d.items(): diff --git a/respodns/dns.py b/respodns/dns.py new file mode 100644 index 0000000..27cdf7d --- /dev/null +++ b/respodns/dns.py @@ -0,0 +1,53 @@ +def detect_gfw(r, ip, check): + # attempt to detect interference from the Great Firewall of China. + #from .ips import china + #if r in china: return True + + # class D or class E, neither of which are correct for a (public?) DNS. + #if int(r.partition(".")[0]) >= 224: return True + + rs = lambda prefix: r.startswith(prefix) + de = lambda suffix: check.domain.endswith(suffix) + hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com") + if rs("31.13.") and not hosted: return True + if rs("66.220."): return True + if rs("69.63."): return True + if rs("69.171.") and not rs("69.171.250."): return True + if rs("74.86."): return True + if rs("75.126."): return True + if r == "64.13.192.74": return True + # more non-facebook GFW stuff: + # 31.13.64.33 + # 31.13.70.1 + # 31.13.70.20 + # 31.13.76.16 + # 31.13.86.1 + # 173.252.110.21 + # 192.99.140.48 + # 199.16.156.40 + # 199.16.158.190 + + return False + +async def getaddrs(server, domain, opts): + from .ip_util import ipkey + from dns.asyncresolver import Resolver + from dns.exception import Timeout + from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers + + res = Resolver(configure=False) + if opts.impatient: + res.timeout = 5 + res.lifetime = 2 + res.nameservers = [server] + try: + ans = await res.resolve(domain, "A", search=False) + except NXDOMAIN: + return ["NXDOMAIN"] + except NoAnswer: + return ["NoAnswer"] + except NoNameservers: + return ["NoNameservers"] + except Timeout: + return ["Timeout"] + return sorted(set(rr.address for rr in ans.rrset), key=ipkey) diff --git a/respodns/ip_util.py b/respodns/ip_util.py new file mode 100644 index 0000000..1318658 --- /dev/null +++ b/respodns/ip_util.py @@ -0,0 +1,26 @@ +import re +ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII) + +def read_ips(f): + # TODO: make async and more robust. (regex pls) + # TODO: does readlines() block if the pipe is left open i.e. user input? + for ip in f.readlines(): + if "#" in ip: + ip, _, _ = ip.partition("#") + ip = ip.strip() + if ip.count(".") != 3: + continue + yield ip + +def addr_to_int(ip): + match = ipv4_pattern.fullmatch(ip) + assert match is not None, row + segs = list(map(int, match.group(1, 2, 3, 4))) + assert all(0 <= seg <= 255 for seg in segs), match.group(0) + numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3] + return numeric + +def ipkey(ip_string): + # this is more lenient than addr_to_int. + segs = [int(s) for s in ip_string.replace(":", ".").split(".")] + return sum(256**(3 - i) * seg for i, seg in enumerate(segs)) diff --git a/respodns/tables.py b/respodns/tables.py new file mode 100644 index 0000000..595f11e --- /dev/null +++ b/respodns/tables.py @@ -0,0 +1,61 @@ +from .util import AttrCheck +import storm.locals as rain + +class TException(rain.Storm, AttrCheck): + __storm_table__ = "Exceptions" + exception_id = rain.Int("ExceptionId", primary=True) + name = rain.Unicode("Name") + fail = rain.Bool("Fail") + +class TExecution(rain.Storm, AttrCheck): + __storm_table__ = "Executions" + execution_id = rain.Int("ExecutionId", primary=True) + start_date = rain.DateTime("StartDate") + finish_date = rain.DateTime("FinishDate") + completed = rain.Bool("Completed") + +class TAddress(rain.Storm, AttrCheck): + __storm_table__ = "Ips" + address_id = rain.Int("IpId", primary=True) + str = rain.Unicode("AsStr") + ip = rain.Int("AsInt") + china = rain.Bool("China") + block_target = rain.Bool("BlockTarget") + server = rain.Bool("Server") + redirect_target = rain.Bool("RedirectTarget") + gfw_target = rain.Bool("GfwTarget") + +class TKind(rain.Storm, AttrCheck): + __storm_table__ = "Kinds" + kind_id = rain.Int("KindId", primary=True) + name = rain.Unicode("Name") + xxid = rain.Int("ExpectExceptionId") + exception = rain.Reference(xxid, "TException.exception_id") + +class TDomain(rain.Storm, AttrCheck): + __storm_table__ = "Domains" + domain_id = rain.Int("DomainId", primary=True) + name = rain.Unicode("Name") + kind_id = rain.Int("KindId") + kind = rain.Reference(kind_id, "TKind.kind_id") + +class TRecord(rain.Storm, AttrCheck): + __storm_table__ = "Records" + row_id = rain.Int("rowid", primary=True) + record_id = rain.Int("RecordId") + address_id = rain.Int("IpId") + address = rain.Reference(address_id, "TAddress.address_id") + +class TMessage(rain.Storm, AttrCheck): + __storm_table__ = "Messages" + message_id = rain.Int("MessageId", primary=True) + execution_id = rain.Int("ExecutionId") + server_id = rain.Int("ServerId") + domain_id = rain.Int("DomainId") + record_id = rain.Int("RecordId") + exception_id = rain.Int("ExceptionId") + execution = rain.Reference(execution_id, "TExecution.execution_id") + server = rain.Reference(server_id, "TAddress.address_id") + domain = rain.Reference(domain_id, "TDomain.domain_id") + #record = rain.Reference(record_id, "TRecord.record_id") + exception = rain.Reference(exception_id, "TException.exception_id") diff --git a/respodns/util.py b/respodns/util.py index 5990e5d..14f1693 100644 --- a/respodns/util.py +++ b/respodns/util.py @@ -1,6 +1,3 @@ -import re -ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII) - rot13_mapping = {} for a, b, c, d in zip("anAN05", "mzMZ49", "naNA50", "zmZM94"): rot13_mapping.update(dict((chr(k), chr(v)) @@ -19,73 +16,6 @@ def nonsense_consistent(domain): length = rng.choices((9, 10, 11, 12), (4, 5, 3, 2))[0] return "".join(rng.choice(ascii_lowercase) for i in range(length)) -def detect_gfw(r, ip, check): - # attempt to detect interference from the Great Firewall of China. - #from .ips import china - #if r in china: return True - - # class D or class E, neither of which are correct for a (public?) DNS. - #if int(r.partition(".")[0]) >= 224: return True - - rs = lambda prefix: r.startswith(prefix) - de = lambda suffix: check.domain.endswith(suffix) - hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com") - if rs("31.13.") and not hosted: return True - if rs("66.220."): return True - if rs("69.63."): return True - if rs("69.171.") and not rs("69.171.250."): return True - if rs("74.86."): return True - if rs("75.126."): return True - if r == "64.13.192.74": return True - # more non-facebook GFW stuff: - # 31.13.64.33 - # 31.13.70.1 - # 31.13.70.20 - # 31.13.76.16 - # 31.13.86.1 - # 173.252.110.21 - # 192.99.140.48 - # 199.16.156.40 - # 199.16.158.190 - - return False - -async def getaddrs(server, domain, opts): - from dns.asyncresolver import Resolver - from dns.exception import Timeout - from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers - #from dns.resolver import Resolver - - res = Resolver(configure=False) - if opts.impatient: - res.timeout = 5 - res.lifetime = 2 - res.nameservers = [server] - try: - #ans = res.resolve(domain, "A", search=False) - ans = await res.resolve(domain, "A", search=False) - except NXDOMAIN: - return ["NXDOMAIN"] - except NoAnswer: - return ["NoAnswer"] - except NoNameservers: - return ["NoNameservers"] - except Timeout: - return ["Timeout"] - #return list(set(rr.address for rr in ans.rrset)) - return sorted(set(rr.address for rr in ans.rrset), key=ipkey) - -def read_ips(f): - # TODO: make async and more robust. (regex pls) - # TODO: does readlines() block if the pipe is left open i.e. user input? - for ip in f.readlines(): - if "#" in ip: - ip, _, _ = ip.partition("#") - ip = ip.strip() - if ip.count(".") != 3: - continue - yield ip - def rot13(s): return "".join(rot13_mapping.get(c, c) for c in s) @@ -101,19 +31,6 @@ def head(n, it): pass return res -def addr_to_int(ip): - match = ipv4_pattern.fullmatch(ip) - assert match is not None, row - segs = list(map(int, match.group(1, 2, 3, 4))) - assert all(0 <= seg <= 255 for seg in segs), match.group(0) - numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3] - return numeric - -def ipkey(ip_string): - # this is more lenient than addr_to_int. - segs = [int(s) for s in ip_string.replace(":", ".").split(".")] - return sum(256**(3 - i) * seg for i, seg in enumerate(segs)) - def taskize(item): from types import CoroutineType from asyncio import Task, create_task