201 lines
4.9 KiB
Python
201 lines
4.9 KiB
Python
|
from asyncio import open_connection, sleep
|
||
|
from collections import namedtuple
|
||
|
from socket import gaierror
|
||
|
from sys import stderr
|
||
|
from time import time
|
||
|
|
||
|
CacheLine = namedtuple("CacheLine", ("time", "code"))
|
||
|
header = ["ip", "time", "code"]
|
||
|
|
||
|
one_month = 30 * 24 * 60 * 60 # in seconds
|
||
|
encoding = "latin-1"
|
||
|
|
||
|
cache_filepath = "ipinfo_cache.csv"
|
||
|
|
||
|
http_cooldown = 0
|
||
|
give_up = False
|
||
|
prepared = False
|
||
|
stored = None
|
||
|
|
||
|
|
||
|
async def http_lookup(ip):
|
||
|
global http_cooldown
|
||
|
|
||
|
host = "ip-api.com"
|
||
|
path = f"/csv/{ip}?fields=2"
|
||
|
|
||
|
err = None
|
||
|
|
||
|
# Quote:
|
||
|
# Your implementation should always check the value of the X-Rl header,
|
||
|
# and if its is 0 you must not send any more requests
|
||
|
# for the duration of X-Ttl in seconds.
|
||
|
while time() < http_cooldown:
|
||
|
wait = http_cooldown - time()
|
||
|
wait = max(wait, 0.1) # wait at least a little bit
|
||
|
await sleep(wait)
|
||
|
|
||
|
query_lines = (
|
||
|
f"GET {path} HTTP/1.1",
|
||
|
f"Host: {host}",
|
||
|
f"Connection: close",
|
||
|
)
|
||
|
query = "\r\n".join(query_lines) + "\r\n\r\n"
|
||
|
|
||
|
reader, writer = await open_connection(host, 80)
|
||
|
|
||
|
writer.write(query.encode(encoding, "strict"))
|
||
|
response = await reader.read()
|
||
|
lines = response.splitlines()
|
||
|
|
||
|
it = iter(lines)
|
||
|
line = next(it)
|
||
|
if line != b"HTTP/1.1 200 OK":
|
||
|
http_cooldown = time() + 60
|
||
|
err = "not ok"
|
||
|
it = () # exhaust iterator (not really)
|
||
|
|
||
|
x_cooldown = None
|
||
|
x_remaining = None
|
||
|
|
||
|
for line in it:
|
||
|
if line == b"":
|
||
|
break
|
||
|
|
||
|
head, _, tail = line.partition(b":")
|
||
|
|
||
|
# do some very basic validation.
|
||
|
if tail[0:1] == b" ":
|
||
|
tail = tail[1:]
|
||
|
else:
|
||
|
err = "bad tail"
|
||
|
break
|
||
|
if head in (b"Date", b"Content-Type", b"Content-Length",
|
||
|
b"Access-Control-Allow-Origin"):
|
||
|
pass
|
||
|
elif head == b"X-Ttl":
|
||
|
if tail.isdigit():
|
||
|
x_cooldown = int(tail)
|
||
|
else:
|
||
|
err = "X-Ttl not integer"
|
||
|
break
|
||
|
elif head == b"X-Rl":
|
||
|
if tail.isdigit():
|
||
|
x_remaining = int(tail)
|
||
|
else:
|
||
|
err = "X-Rl not integer"
|
||
|
break
|
||
|
|
||
|
for i, line in enumerate(it):
|
||
|
if i == 0:
|
||
|
code = line
|
||
|
else:
|
||
|
err = "too many lines"
|
||
|
break
|
||
|
|
||
|
writer.close()
|
||
|
|
||
|
if x_remaining == 0:
|
||
|
http_cooldown = time() + x_cooldown
|
||
|
http_cooldown += 1.0 # still too frequent according to them
|
||
|
|
||
|
if err:
|
||
|
return None, err
|
||
|
else:
|
||
|
return code, None
|
||
|
|
||
|
|
||
|
async def lookup(ip, timestamp):
|
||
|
global give_up
|
||
|
if give_up:
|
||
|
return None
|
||
|
|
||
|
try:
|
||
|
code, err = await http_lookup(ip)
|
||
|
if err:
|
||
|
# retry once in case of rate-limiting
|
||
|
code, err = await http_lookup(ip)
|
||
|
if err:
|
||
|
return None
|
||
|
except gaierror:
|
||
|
give_up = True
|
||
|
except OSError:
|
||
|
give_up = True
|
||
|
|
||
|
code = code.decode(encoding, "ignore")
|
||
|
if code == "":
|
||
|
code = "--"
|
||
|
if len(code) != 2:
|
||
|
return None
|
||
|
info = CacheLine(timestamp, code)
|
||
|
return info
|
||
|
|
||
|
|
||
|
def prepare():
|
||
|
from csv import reader, writer
|
||
|
from os.path import exists
|
||
|
|
||
|
global stored
|
||
|
stored = dict()
|
||
|
|
||
|
if not exists(cache_filepath):
|
||
|
with open(cache_filepath, "w") as f:
|
||
|
handle = writer(f)
|
||
|
handle.writerow(header)
|
||
|
return
|
||
|
|
||
|
with open(cache_filepath, "r", newline="", encoding="utf-8") as f:
|
||
|
for i, row in enumerate(reader(f)):
|
||
|
if i == 0:
|
||
|
assert row == header, row
|
||
|
continue
|
||
|
ip, time, code = row[0], float(row[1]), row[2]
|
||
|
info = CacheLine(time, code)
|
||
|
stored[ip] = info
|
||
|
|
||
|
|
||
|
def flush():
|
||
|
from csv import writer
|
||
|
if not stored:
|
||
|
return
|
||
|
with open(cache_filepath, "w", newline="", encoding="utf-8") as f:
|
||
|
handle = writer(f)
|
||
|
handle.writerow(header)
|
||
|
for ip, info in stored.items():
|
||
|
timestr = "{:.2f}".format(info.time)
|
||
|
handle.writerow([ip, timestr, info.code])
|
||
|
|
||
|
|
||
|
def cache(ip, info=None, timestamp=None, expiry=one_month):
|
||
|
global stored
|
||
|
if stored is None:
|
||
|
prepare()
|
||
|
|
||
|
now = time() if timestamp is None else timestamp
|
||
|
|
||
|
if info is None:
|
||
|
cached = stored.get(ip, None)
|
||
|
if cached is None:
|
||
|
return None
|
||
|
if now > cached.time + expiry:
|
||
|
return None
|
||
|
return cached
|
||
|
else:
|
||
|
assert isinstance(info, CacheLine)
|
||
|
stored[ip] = info
|
||
|
|
||
|
|
||
|
async def find_country(ip, db=None):
|
||
|
now = time()
|
||
|
info = cache(ip, timestamp=now)
|
||
|
if info is None:
|
||
|
info = await lookup(ip, now)
|
||
|
if info is None:
|
||
|
return None
|
||
|
cache(ip, info)
|
||
|
if db is not None:
|
||
|
if db.country_code(ip) != info.code:
|
||
|
assert info.code is not None
|
||
|
db.country_code(ip, info.code)
|
||
|
return info.code
|