notwa/respodns
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

further reorganization

Browse source
This commit is contained in:
Connor Olding 2020-08-29 15:04:56 +02:00
parent ecb2596090
commit 85045f397f
6 changed files with 146 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
respodns/__main__.py
View file

@ -1,9 +1,11 @@
#!/usr/bin/env python3
from .db import RespoDB
from .dns import getaddrs, detect_gfw
from .ip_util import addr_to_int, read_ips
from .ips import blocks, is_bogon
from .structs import Options, Entry
from .util import right_now, read_ips, getaddrs, detect_gfw, make_pooler
from .util import right_now, make_pooler
from asyncio import run, sleep
from sys import argv, stdin, stderr, exit
import respodns.checks as chk

63
respodns/db.py
View file

@ -1,6 +1,8 @@
from .sql import create_table_statements, create_view_statements
from .sql import table_triggers
from .util import addr_to_int, AttrCheck
from .tables import TException, TExecution, TAddress
from .tables import TKind, TDomain, TRecord, TMessage
from .ip_util import addr_to_int
import storm.locals as rain
class Execution:
@ -18,65 +20,6 @@ class Execution:
completed = exc_type is None
self.db.finish_execution(self.execution, right_now(), completed)
class TException(rain.Storm, AttrCheck):
__storm_table__ = "Exceptions"
exception_id = rain.Int("ExceptionId", primary=True)
name = rain.Unicode("Name")
fail = rain.Bool("Fail")
class TExecution(rain.Storm, AttrCheck):
__storm_table__ = "Executions"
execution_id = rain.Int("ExecutionId", primary=True)
start_date = rain.DateTime("StartDate")
finish_date = rain.DateTime("FinishDate")
completed = rain.Bool("Completed")
class TAddress(rain.Storm, AttrCheck):
__storm_table__ = "Ips"
address_id = rain.Int("IpId", primary=True)
str = rain.Unicode("AsStr")
ip = rain.Int("AsInt")
china = rain.Bool("China")
block_target = rain.Bool("BlockTarget")
server = rain.Bool("Server")
redirect_target = rain.Bool("RedirectTarget")
gfw_target = rain.Bool("GfwTarget")
class TKind(rain.Storm, AttrCheck):
__storm_table__ = "Kinds"
kind_id = rain.Int("KindId", primary=True)
name = rain.Unicode("Name")
xxid = rain.Int("ExpectExceptionId")
exception = rain.Reference(xxid, "TException.exception_id")
class TDomain(rain.Storm, AttrCheck):
__storm_table__ = "Domains"
domain_id = rain.Int("DomainId", primary=True)
name = rain.Unicode("Name")
kind_id = rain.Int("KindId")
kind = rain.Reference(kind_id, "TKind.kind_id")
class TRecord(rain.Storm, AttrCheck):
__storm_table__ = "Records"
row_id = rain.Int("rowid", primary=True)
record_id = rain.Int("RecordId")
address_id = rain.Int("IpId")
address = rain.Reference(address_id, "TAddress.address_id")
class TMessage(rain.Storm, AttrCheck):
__storm_table__ = "Messages"
message_id = rain.Int("MessageId", primary=True)
execution_id = rain.Int("ExecutionId")
server_id = rain.Int("ServerId")
domain_id = rain.Int("DomainId")
record_id = rain.Int("RecordId")
exception_id = rain.Int("ExceptionId")
execution = rain.Reference(execution_id, "TExecution.execution_id")
server = rain.Reference(server_id, "TAddress.address_id")
domain = rain.Reference(domain_id, "TDomain.domain_id")
#record = rain.Reference(record_id, "TRecord.record_id")
exception = rain.Reference(exception_id, "TException.exception_id")
def apply_properties(obj, d):
from storm.properties import PropertyColumn
for k, v in d.items():

53
respodns/dns.py Normal file
View file

@ -0,0 +1,53 @@
def detect_gfw(r, ip, check):
# attempt to detect interference from the Great Firewall of China.
#from .ips import china
#if r in china: return True
# class D or class E, neither of which are correct for a (public?) DNS.
#if int(r.partition(".")[0]) >= 224: return True
rs = lambda prefix: r.startswith(prefix)
de = lambda suffix: check.domain.endswith(suffix)
hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com")
if rs("31.13.") and not hosted: return True
if rs("66.220."): return True
if rs("69.63."): return True
if rs("69.171.") and not rs("69.171.250."): return True
if rs("74.86."): return True
if rs("75.126."): return True
if r == "64.13.192.74": return True
# more non-facebook GFW stuff:
# 31.13.64.33
# 31.13.70.1
# 31.13.70.20
# 31.13.76.16
# 31.13.86.1
# 173.252.110.21
# 192.99.140.48
# 199.16.156.40
# 199.16.158.190
return False
async def getaddrs(server, domain, opts):
from .ip_util import ipkey
from dns.asyncresolver import Resolver
from dns.exception import Timeout
from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers
res = Resolver(configure=False)
if opts.impatient:
res.timeout = 5
res.lifetime = 2
res.nameservers = [server]
try:
ans = await res.resolve(domain, "A", search=False)
except NXDOMAIN:
return ["NXDOMAIN"]
except NoAnswer:
return ["NoAnswer"]
except NoNameservers:
return ["NoNameservers"]
except Timeout:
return ["Timeout"]
return sorted(set(rr.address for rr in ans.rrset), key=ipkey)

26
respodns/ip_util.py Normal file
View file

@ -0,0 +1,26 @@
import re
ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII)
def read_ips(f):
# TODO: make async and more robust. (regex pls)
# TODO: does readlines() block if the pipe is left open i.e. user input?
for ip in f.readlines():
if "#" in ip:
ip, _, _ = ip.partition("#")
ip = ip.strip()
if ip.count(".") != 3:
continue
yield ip
def addr_to_int(ip):
match = ipv4_pattern.fullmatch(ip)
assert match is not None, row
segs = list(map(int, match.group(1, 2, 3, 4)))
assert all(0 <= seg <= 255 for seg in segs), match.group(0)
numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3]
return numeric
def ipkey(ip_string):
# this is more lenient than addr_to_int.
segs = [int(s) for s in ip_string.replace(":", ".").split(".")]
return sum(256**(3 - i) * seg for i, seg in enumerate(segs))

61
respodns/tables.py Normal file
View file

@ -0,0 +1,61 @@
from .util import AttrCheck
import storm.locals as rain
class TException(rain.Storm, AttrCheck):
__storm_table__ = "Exceptions"
exception_id = rain.Int("ExceptionId", primary=True)
name = rain.Unicode("Name")
fail = rain.Bool("Fail")
class TExecution(rain.Storm, AttrCheck):
__storm_table__ = "Executions"
execution_id = rain.Int("ExecutionId", primary=True)
start_date = rain.DateTime("StartDate")
finish_date = rain.DateTime("FinishDate")
completed = rain.Bool("Completed")
class TAddress(rain.Storm, AttrCheck):
__storm_table__ = "Ips"
address_id = rain.Int("IpId", primary=True)
str = rain.Unicode("AsStr")
ip = rain.Int("AsInt")
china = rain.Bool("China")
block_target = rain.Bool("BlockTarget")
server = rain.Bool("Server")
redirect_target = rain.Bool("RedirectTarget")
gfw_target = rain.Bool("GfwTarget")
class TKind(rain.Storm, AttrCheck):
__storm_table__ = "Kinds"
kind_id = rain.Int("KindId", primary=True)
name = rain.Unicode("Name")
xxid = rain.Int("ExpectExceptionId")
exception = rain.Reference(xxid, "TException.exception_id")
class TDomain(rain.Storm, AttrCheck):
__storm_table__ = "Domains"
domain_id = rain.Int("DomainId", primary=True)
name = rain.Unicode("Name")
kind_id = rain.Int("KindId")
kind = rain.Reference(kind_id, "TKind.kind_id")
class TRecord(rain.Storm, AttrCheck):
__storm_table__ = "Records"
row_id = rain.Int("rowid", primary=True)
record_id = rain.Int("RecordId")
address_id = rain.Int("IpId")
address = rain.Reference(address_id, "TAddress.address_id")
class TMessage(rain.Storm, AttrCheck):
__storm_table__ = "Messages"
message_id = rain.Int("MessageId", primary=True)
execution_id = rain.Int("ExecutionId")
server_id = rain.Int("ServerId")
domain_id = rain.Int("DomainId")
record_id = rain.Int("RecordId")
exception_id = rain.Int("ExceptionId")
execution = rain.Reference(execution_id, "TExecution.execution_id")
server = rain.Reference(server_id, "TAddress.address_id")
domain = rain.Reference(domain_id, "TDomain.domain_id")
#record = rain.Reference(record_id, "TRecord.record_id")
exception = rain.Reference(exception_id, "TException.exception_id")

83
respodns/util.py
View file

@ -1,6 +1,3 @@
import re
ipv4_pattern = re.compile("(\d+)\.(\d+)\.(\d+)\.(\d+)", re.ASCII)
rot13_mapping = {}
for a, b, c, d in zip("anAN05", "mzMZ49", "naNA50", "zmZM94"):
rot13_mapping.update(dict((chr(k), chr(v))
@ -19,73 +16,6 @@ def nonsense_consistent(domain):
length = rng.choices((9, 10, 11, 12), (4, 5, 3, 2))[0]
return "".join(rng.choice(ascii_lowercase) for i in range(length))
def detect_gfw(r, ip, check):
# attempt to detect interference from the Great Firewall of China.
#from .ips import china
#if r in china: return True
# class D or class E, neither of which are correct for a (public?) DNS.
#if int(r.partition(".")[0]) >= 224: return True
rs = lambda prefix: r.startswith(prefix)
de = lambda suffix: check.domain.endswith(suffix)
hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com")
if rs("31.13.") and not hosted: return True
if rs("66.220."): return True
if rs("69.63."): return True
if rs("69.171.") and not rs("69.171.250."): return True
if rs("74.86."): return True
if rs("75.126."): return True
if r == "64.13.192.74": return True
# more non-facebook GFW stuff:
# 31.13.64.33
# 31.13.70.1
# 31.13.70.20
# 31.13.76.16
# 31.13.86.1
# 173.252.110.21
# 192.99.140.48
# 199.16.156.40
# 199.16.158.190
return False
async def getaddrs(server, domain, opts):
from dns.asyncresolver import Resolver
from dns.exception import Timeout
from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers
#from dns.resolver import Resolver
res = Resolver(configure=False)
if opts.impatient:
res.timeout = 5
res.lifetime = 2
res.nameservers = [server]
try:
#ans = res.resolve(domain, "A", search=False)
ans = await res.resolve(domain, "A", search=False)
except NXDOMAIN:
return ["NXDOMAIN"]
except NoAnswer:
return ["NoAnswer"]
except NoNameservers:
return ["NoNameservers"]
except Timeout:
return ["Timeout"]
#return list(set(rr.address for rr in ans.rrset))
return sorted(set(rr.address for rr in ans.rrset), key=ipkey)
def read_ips(f):
# TODO: make async and more robust. (regex pls)
# TODO: does readlines() block if the pipe is left open i.e. user input?
for ip in f.readlines():
if "#" in ip:
ip, _, _ = ip.partition("#")
ip = ip.strip()
if ip.count(".") != 3:
continue
yield ip
def rot13(s):
return "".join(rot13_mapping.get(c, c) for c in s)
@ -101,19 +31,6 @@ def head(n, it):
pass
return res
def addr_to_int(ip):
match = ipv4_pattern.fullmatch(ip)
assert match is not None, row
segs = list(map(int, match.group(1, 2, 3, 4)))
assert all(0 <= seg <= 255 for seg in segs), match.group(0)
numeric = segs[0] << 24 | segs[1] << 16 | segs[2] << 8 | segs[3]
return numeric
def ipkey(ip_string):
# this is more lenient than addr_to_int.
segs = [int(s) for s in ip_string.replace(":", ".").split(".")]
return sum(256**(3 - i) * seg for i, seg in enumerate(segs))
def taskize(item):
from types import CoroutineType
from asyncio import Task, create_task