285 lines
8.1 KiB
Python
285 lines
8.1 KiB
Python
from .structs import Options
|
|
|
|
|
|
def detect_gfw(r, ip, check):
|
|
# attempt to detect interference from the Great Firewall of China.
|
|
from .ips import gfw_ips
|
|
|
|
def rs(prefix):
|
|
return r.startswith(prefix)
|
|
|
|
def de(suffix):
|
|
return check.domain.endswith(suffix)
|
|
|
|
hosted = de("facebook.com") or de("instagram.com") or de("whatsapp.com")
|
|
|
|
if rs("31.13.") and not hosted:
|
|
return True
|
|
|
|
if rs("69.171.") and not rs("69.171.250."):
|
|
return True
|
|
|
|
if rs("108.160."):
|
|
return True
|
|
|
|
if r in gfw_ips:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
async def getaddrs(server, domain, opts):
|
|
from .ip_util import ipkey
|
|
from dns.asyncresolver import Resolver
|
|
from dns.exception import Timeout
|
|
from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers
|
|
|
|
res = Resolver(configure=False)
|
|
if opts.impatient:
|
|
res.timeout = 5
|
|
res.lifetime = 2
|
|
else:
|
|
res.timeout = 3
|
|
res.lifetime = 9
|
|
res.nameservers = [server]
|
|
try:
|
|
ans = await res.resolve(domain, "A", search=False)
|
|
except NXDOMAIN:
|
|
return ["NXDOMAIN"]
|
|
except NoAnswer:
|
|
return ["NoAnswer"]
|
|
except NoNameservers:
|
|
return ["NoNameservers"]
|
|
except Timeout:
|
|
return ["Timeout"]
|
|
return sorted(set(rr.address for rr in ans.rrset), key=ipkey)
|
|
|
|
|
|
def process_result(res, ip, check, opts: Options):
|
|
from .ips import is_bogon, is_block_target
|
|
from .util import right_now
|
|
from .structs import Entry
|
|
|
|
# TODO: get more accurate times by inserting start-end into getaddrs.
|
|
now = right_now()
|
|
assert len(res) > 0
|
|
reason = None
|
|
|
|
if "Timeout" in res:
|
|
reason = "timeout"
|
|
|
|
elif check.kind.startswith("bad"):
|
|
reason = "okay" if "NXDOMAIN" in res else "redirect"
|
|
|
|
elif any(r == ip for r in res):
|
|
# server returned its own IP, must be a block, unless
|
|
# one of our checks actually contains a domain that also hosts a DNS.
|
|
reason = "block"
|
|
|
|
elif any(is_bogon(r) or is_block_target(r) for r in res):
|
|
reason = "block"
|
|
|
|
elif not any(len(r) > 0 and r[0].isdigit() for r in res):
|
|
# TODO: check for no alias on common.
|
|
reason = "missing"
|
|
|
|
else:
|
|
for r in res:
|
|
if len(r) == 0 or not r[0].isdigit():
|
|
continue
|
|
if detect_gfw(r, ip, check):
|
|
reason = "gfw"
|
|
break
|
|
else:
|
|
reason = "okay"
|
|
|
|
assert reason is not None, (res, ip, check)
|
|
|
|
addrs = list(filter(lambda r: len(r) > 0 and r[0].isdigit(), res))
|
|
exception = res[0] if len(addrs) == 0 else None
|
|
|
|
return Entry(
|
|
date=now,
|
|
success=reason == "okay" or check.kind == "ignore",
|
|
server=ip,
|
|
kind=check.kind,
|
|
domain=check.domain,
|
|
exception=exception,
|
|
addrs=addrs,
|
|
reason=reason,
|
|
execution=opts.execution,
|
|
)
|
|
|
|
|
|
async def try_ip(db, server_ip, checks, opts: Options):
|
|
from .util import make_pooler
|
|
from asyncio import sleep
|
|
|
|
entries = []
|
|
deferred = []
|
|
|
|
success = True
|
|
|
|
def maybe_put_ip(ip):
|
|
from asyncio import QueueFull
|
|
try:
|
|
opts.ips.put_nowait(ip)
|
|
except QueueFull:
|
|
deferred.append(ip)
|
|
|
|
def finisher(done, pending):
|
|
nonlocal success
|
|
for task in done:
|
|
res, ip, check = task.result()
|
|
entry = process_result(res, ip, check, opts)
|
|
map(maybe_put_ip, entry.addrs)
|
|
entries.append(entry)
|
|
if not entry.success:
|
|
if opts.early_stopping and success: # only cancel once
|
|
for pend in pending:
|
|
# FIXME: this can still, somehow,
|
|
# cancel the main function.
|
|
pend.cancel()
|
|
success = False
|
|
|
|
pooler = make_pooler(opts.domain_simul, finisher)
|
|
|
|
async def getaddrs_wrapper(ip, check):
|
|
# NOTE: could put right_now() stuff here!
|
|
# TODO: add duration field given in milliseconds (integer)
|
|
# by subtracting start and end datetimes.
|
|
res = await getaddrs(ip, check.domain, opts)
|
|
return res, ip, check
|
|
|
|
for i, check in enumerate(checks):
|
|
first = i == 0
|
|
if not first:
|
|
await sleep(opts.domain_wait)
|
|
await pooler(getaddrs_wrapper(server_ip, check))
|
|
if first:
|
|
# limit to one connection for the first check.
|
|
await pooler()
|
|
if not success:
|
|
if opts.early_stopping or first:
|
|
break
|
|
else:
|
|
await pooler()
|
|
|
|
for ip in deferred:
|
|
await opts.ips.put(ip)
|
|
|
|
if not opts.dry:
|
|
for entry in entries:
|
|
db.push_entry(entry)
|
|
db.commit()
|
|
|
|
if not success:
|
|
first_failure = None
|
|
assert len(entries) > 0
|
|
for entry in entries:
|
|
if not entry.success:
|
|
first_failure = entry
|
|
break
|
|
else:
|
|
assert 0, ("no failures found:", entries)
|
|
return server_ip, first_failure
|
|
return server_ip, None
|
|
|
|
|
|
async def sync_database(db, opts: Options):
|
|
from .ips import china, blocks
|
|
|
|
# TODO: handle addresses that were removed from respodns.ips.china.
|
|
for ips, kw in ((china, "china"), (blocks, "block_target")):
|
|
for ip in ips:
|
|
kwargs = {kw: True}
|
|
if db is not None:
|
|
db.modify_address(ip, **kwargs)
|
|
await opts.ips.put(ip)
|
|
|
|
|
|
async def locate_ips(db, opts: Options):
|
|
from time import time
|
|
seen = set()
|
|
last_save = time()
|
|
while (ip := await opts.ips.get()) is not None:
|
|
if opts.ipinfo is not None and ip not in seen:
|
|
seen.add(ip)
|
|
code = await opts.ipinfo.find_country(ip)
|
|
if db is not None:
|
|
db.modify_address(ip, country_code=code)
|
|
if time() >= last_save + 10.0: # only flush occasionally
|
|
opts.ipinfo.flush()
|
|
last_save = time()
|
|
opts.ipinfo.flush()
|
|
|
|
|
|
async def main(db, filepaths, checks, opts: Options):
|
|
from .util import make_pooler
|
|
from asyncio import sleep, create_task, Queue
|
|
from sys import stdin, stderr
|
|
|
|
opts.ips = Queue()
|
|
syncing = create_task(sync_database(db, opts))
|
|
geoip = create_task(locate_ips(db, opts))
|
|
|
|
def finisher(done, pending):
|
|
for task in done:
|
|
ip, first_failure = task.result()
|
|
if first_failure is None:
|
|
print(ip)
|
|
elif opts.dry:
|
|
ff = first_failure
|
|
if ff.kind in ("shock", "adware"):
|
|
print(ip, ff.reason, ff.kind, sep="\t")
|
|
else:
|
|
print(ip, ff.reason, ff.kind, ff.domain, sep="\t")
|
|
|
|
pooler = make_pooler(opts.ip_simul, finisher)
|
|
|
|
seen = 0
|
|
|
|
async def process(ip, total=None):
|
|
nonlocal seen
|
|
first = seen == 0
|
|
seen += 1
|
|
if opts.progress:
|
|
if total is None:
|
|
print(f"#{seen}: {ip}", file=stderr)
|
|
else:
|
|
print(f"#{seen}/{total}: {ip}", file=stderr)
|
|
stderr.flush()
|
|
if not first:
|
|
await sleep(opts.ip_wait)
|
|
await opts.ips.put(ip)
|
|
await pooler(try_ip(db, ip, checks, opts))
|
|
|
|
if opts.blocking_file_io:
|
|
from .ip_util import read_ips
|
|
for filepath in filepaths:
|
|
f = stdin if filepath == "" else open(filepath, "r")
|
|
for ip in read_ips(f):
|
|
await process(ip)
|
|
if f != stdin:
|
|
f.close()
|
|
|
|
else:
|
|
from .ip_util import IpReader
|
|
fps = [stdin if fp == "" else fp for fp in filepaths]
|
|
with IpReader(*fps) as reader:
|
|
async for ip in reader:
|
|
await process(ip, reader.total)
|
|
|
|
if seen == 0:
|
|
# no IPs were provided. refresh all the country codes instead.
|
|
all_ips = db.all_ips()
|
|
for i, ip in enumerate(all_ips):
|
|
if opts.progress:
|
|
print(f"#{i + 1}/{len(all_ips)}: {ip}", file=stderr)
|
|
await opts.ips.put(ip)
|
|
|
|
await pooler()
|
|
await syncing
|
|
await opts.ips.put(None) # end of queue
|
|
await geoip
|